openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
885 lines (884 loc) • 33 kB
JavaScript
import { s as toErrorObject } from "./error-coercion-CwIUUFnu.mjs";
import { i as receiveWorkerMemoryPort, n as createCpuTrackedWorker, r as markWorkerRetirement, s as resolveRuntimeWorkerThreadExecArgv, t as attributeWorkerToPool } from "./worker-cpu-BzVGEcKM.mjs";
import { D as resolveTimerTimeoutMs, M as isRecord } from "./utils-Dor5uhSv.mjs";
import { d as resolveGlobalSingleton } from "./redact-BwKH5fIe.mjs";
import { t as createDeferredCore } from "./deferred-D0La5CRk.mjs";
import { t as runBestEffortCleanup } from "./non-fatal-cleanup-BoCq8AND.mjs";
import { availableParallelism } from "node:os";
import { AsyncLocalStorage } from "node:async_hooks";
import { channel } from "node:diagnostics_channel";
import { MessageChannel } from "node:worker_threads";
/** Shared compute admission; database and generation workers keep their own ordering. */
function getWorkerComputeCapacity() {
return resolveGlobalSingleton(Symbol.for("openclaw.workerComputeCapacity"), () => {
const limit = Math.max(1, availableParallelism() - 1);
const active = /* @__PURE__ */ new Set();
const waiting = /* @__PURE__ */ new Set();
let pendingTasks = 0;
let pendingBytes = 0;
let draining = false;
const requestCheckpoints = () => {
let demand = waiting.size;
if (demand) {
for (const permit of active) if (permit.requestCheckpoint() && --demand === 0) break;
}
};
const drain = () => {
if (draining) return;
draining = true;
try {
while (active.size < limit && waiting.size) {
const next = waiting.values().next().value;
waiting.delete(next);
next();
}
} finally {
draining = false;
}
requestCheckpoints();
};
return {
getSnapshot: () => ({
limit,
active: active.size,
waitingPools: waiting.size,
pendingTasks,
pendingBytes
}),
admit(bytes) {
if (pendingTasks >= 128 || pendingBytes + bytes > 268435456) return false;
pendingTasks++;
pendingBytes += bytes;
return true;
},
finish(bytes) {
pendingTasks--;
pendingBytes -= bytes;
},
acquire(resume, requestCheckpoint) {
if (active.size >= limit || !draining && waiting.size) {
waiting.add(resume);
requestCheckpoints();
return;
}
const permit = { requestCheckpoint };
active.add(permit);
return permit;
},
release(permit) {
active.delete(permit);
drain();
},
remove(resume) {
waiting.delete(resume);
},
requestCheckpoints
};
});
}
//#endregion
//#region src/infra/worker-task-native-sections.ts
const CANCELLED = 1;
const SECTION = 2;
resolveGlobalSingleton(Symbol.for("openclaw.workerTaskNativeSection"), () => new AsyncLocalStorage());
function createWorkerNativeSectionState() {
return new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
}
function cancelWorkerNativeSections(state) {
Atomics.or(state, 0, CANCELLED);
Atomics.notify(state, 0);
}
function waitForWorkerNativeSections(state) {
if (Atomics.load(state, 0) < SECTION) return;
return (async () => {
for (;;) {
const observed = Atomics.load(state, 0);
if (observed < SECTION) return;
await Atomics.waitAsync(state, 0, observed).value;
}
})();
}
function releaseWorkerNativeSectionsOnExit(state) {
Atomics.store(state, 0, CANCELLED);
Atomics.notify(state, 0);
}
//#endregion
//#region src/infra/worker-task-pool-owned.ts
async function joinOwnedWorkerTasks(closures) {
const errors = (await Promise.allSettled(closures)).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
if (errors.length === 1) throw errors[0];
if (errors.length > 1) throw new AggregateError(errors, "Worker task cleanup failed");
}
function joinOwnedWorkerTask(task, settlement, retire = false) {
const closing = closeOwnedWorkerTask(task, settlement, retire);
return closing.catch((error) => {
if (task.owner?.closing === closing) task.owner.closing = void 0;
throw error;
}).finally(() => {
if (task.owner?.closed) {
const slot = task.slot;
task.slot = void 0;
settlement.release(task, slot);
}
});
}
function closeOwnedWorkerTask(task, settlement, retire = false) {
const owner = task.owner;
if (!owner) return Promise.resolve();
if (owner.closed) return owner.closing ?? Promise.resolve();
owner.retire ||= retire;
if (!task.done) settlement.cancel(task);
return owner.closing ??= Promise.resolve().then(async () => {
await task.preparation?.promise;
const slot = task.slot;
if (slot && owner.retire) await settlement.retire(slot);
settlement.detach(task);
try {
owner.complete?.();
} finally {
owner.closed = true;
owner.complete = void 0;
task.input = void 0;
task.options = {};
task.abort = () => {};
}
});
}
//#endregion
//#region src/infra/worker-task-pool-completion.ts
const taskDiagnostics = channel("openclaw.worker.task");
function joinWorkerTaskPreparationCleanups(completion, artifactCleanup) {
return joinOwnedWorkerTasks([...[...completion.preparationCleanups].map(([task, cleanup]) => cleanup.finally(() => {
if (completion.preparationCleanups.delete(task)) completion.releaseAdmission(task);
})), artifactCleanup.then(() => void 0)]);
}
function notifyExecutionSettled(task) {
try {
if (!task.executionNotified) {
task.executionNotified = true;
task.options.onExecutionSettled?.({ retired: task.slot?.retired === true });
}
} catch (error) {
return toErrorObject(error, "worker settlement receipt failed");
}
}
function createWorkerTaskCompletion(task, completion, error, value) {
const preparation = task.owner ? void 0 : task.preparation;
let settlementError;
const complete = () => {
if (!task.owner && !preparation) completion.releaseAdmission(task);
return completeWorkerTask(task, completion, error, value, settlementError);
};
if (!preparation) return complete;
const cleanup = createDeferredCore();
completion.preparationCleanups.set(task, cleanup.promise);
cleanup.promise.catch(() => void 0);
return () => {
settlementError = task.runInContext(() => notifyExecutionSettled(task));
if (error) task.reject(error);
preparation.promise.then(() => {
const failure = complete();
if (failure) throw failure;
completion.preparationCleanups.delete(task);
completion.releaseAdmission(task);
cleanup.resolve();
}).catch(cleanup.reject);
};
}
function completeWorkerTask(task, completion, error, value, settlementError) {
return task.runInContext(() => {
let completionError = error ?? settlementError;
const cleanupErrors = settlementError ? [settlementError] : [];
try {
if (!task.inputConsumed) {
task.inputConsumed = true;
task.options.onInputConsumed?.();
}
const release = task.exchange?.onConsumed;
task.exchange = void 0;
release?.();
} catch (releaseError) {
const cleanupError = toErrorObject(releaseError, "worker input release failed");
cleanupErrors.push(cleanupError);
completionError ??= cleanupError;
}
const receiptError = notifyExecutionSettled(task);
if (receiptError) {
cleanupErrors.push(receiptError);
completionError ??= receiptError;
}
const permit = task.computePermit;
task.computePermit = void 0;
if (permit) completion.releaseCompute(permit);
if (taskDiagnostics.hasSubscribers) {
const now = performance.now();
taskDiagnostics.publish({
...completion.diagnostics(),
outcome: completionError ? "failed" : "ok",
queueMs: (task.startedAt ?? now) - task.enqueuedAt,
preparationMs: task.startedAt === void 0 ? 0 : (task.preparedAt ?? now) - task.startedAt,
runMs: task.preparedAt === void 0 ? 0 : now - task.preparedAt,
transferMs: task.transferMs
});
}
const firstCleanupError = cleanupErrors[0];
const cleanupError = cleanupErrors.length > 1 ? new AggregateError(cleanupErrors, "Worker task cleanup failed", { cause: firstCleanupError }) : firstCleanupError;
if (task.owner) {
if (cleanupError) throw cleanupError;
} else if (completionError) task.reject(completionError);
else task.resolve(value);
return cleanupError;
});
}
//#endregion
//#region src/infra/worker-task-pool-resources.ts
/** Keep each pool slot referenced until its resource cleanup settles. */
async function closeWorkerPoolResources(slots, resourceClosures, key) {
const errors = (await Promise.allSettled([...slots].map((slot) => {
if (slot.retiring) return slot.retiring;
const worker = slot.worker;
if (!worker) return Promise.resolve();
const closure = resourceClosures.get(worker) ?? { pending: 0 };
closure.pending += 1;
resourceClosures.set(worker, closure);
worker.ref();
return closeWorkerTaskResources(worker, closure, () => slot.retiring, key).finally(() => {
closure.pending -= 1;
if (!closure.pending && !slot.task && !slot.retiring) worker.unref();
});
}))).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
if (errors.length) throw new AggregateError(errors, "Worker resource cleanup failed");
}
/** Concurrent resource closes share one listener, retained only until their last receipt. */
function observeResourceWorkerExit(worker, closures, finish) {
let exit = closures.exit;
if (!exit) {
const waiters = /* @__PURE__ */ new Set();
exit = {
waiters,
receive: () => {
for (const waiter of waiters) waiter();
}
};
closures.exit = exit;
worker.once("exit", exit.receive);
}
exit.waiters.add(finish);
return () => {
exit.waiters.delete(finish);
if (exit.waiters.size === 0) {
worker.removeListener("exit", exit.receive);
if (closures.exit === exit) closures.exit = void 0;
}
};
}
/** The pool keeps the Worker referenced until its cleanup receipt or confirmed native exit. */
function closeWorkerTaskResources(worker, closures, readRetirement, key) {
const { port1, port2 } = new MessageChannel();
return new Promise((resolve, reject) => {
let settled = false;
const finish = (error) => {
if (settled) return;
settled = true;
stopObservingExit();
port1.close();
port2.close();
if (error) reject(error);
else resolve();
};
const stopObservingExit = observeResourceWorkerExit(worker, closures, () => finish());
port1.once("message", (reply) => {
finish(isRecord(reply) && reply.ok === true ? void 0 : new Error(isRecord(reply) && typeof reply.error === "string" ? reply.error : "Invalid worker resource cleanup receipt"));
});
port1.once("messageerror", (error) => finish(toErrorObject(error, "Worker resource cleanup receipt could not be decoded")));
port1.once("close", () => {
if (settled) return;
const retirement = readRetirement();
if (retirement) retirement.then(() => finish(), (error) => finish(toErrorObject(error, "Worker resource cleanup retirement failed")));
else finish(/* @__PURE__ */ new Error("Worker resource cleanup closed without a receipt"));
});
try {
worker.postMessage({
closeResource: true,
key,
resourcePort: port2
}, [port2]);
} catch (error) {
finish(toErrorObject(error, "Worker resource cleanup could not be sent"));
}
});
}
//#endregion
//#region src/infra/worker-task-pool-retirement.ts
const WORKER_WARM_WINDOW_MS = 3e5;
function createWorkerTaskPoolRetirement({ slots, options, runInContext, dispatch }) {
const artifactCleanups = /* @__PURE__ */ new Set();
let lastIdleRetirementAt = -Infinity;
let warmSlot;
const setTimeoutFn = setTimeout;
const clearTimeoutFn = clearTimeout;
const now = performance.now.bind(performance);
const clearIdle = (slot) => clearTimeoutFn(slot.idleTimer);
function retire(slot, reason = "closed") {
if (reason === "idle_timeout") lastIdleRetirementAt = now();
else if (reason === "rotation") lastIdleRetirementAt = -Infinity;
if (warmSlot === slot) warmSlot = void 0;
if (slot.worker) markWorkerRetirement(slot.worker, reason);
clearIdle(slot);
cancelWorkerNativeSections(slot.nativeSections);
return slot.retiring ??= Promise.resolve().then(async () => {
if (slot.worker) {
markWorkerRetirement(slot.worker, reason);
const settlement = waitForWorkerNativeSections(slot.nativeSections);
if (settlement) await settlement;
await slot.worker.terminate();
}
slot.retired = true;
}).catch((error) => {
try {
Promise.resolve(options.onRetirementFailure?.(error)).catch(() => void 0);
} catch {}
throw error;
}).then(() => {
const releaseResources = slot.releaseResources;
if (releaseResources) runInContext(() => {
const cleanup = runBestEffortCleanup({
cleanup: releaseResources,
onError: (error) => process.emitWarning(`Worker task resource release failed: ${String(error)}`)
});
artifactCleanups.add(cleanup);
cleanup.then(() => artifactCleanups.delete(cleanup));
});
slot.worker?.removeAllListeners();
slots.delete(slot);
for (const complete of slot.completions ?? []) complete();
slot.completions = void 0;
dispatch();
}).catch((error) => {
slot.retirementFailed = true;
slot.retiring = void 0;
throw error;
});
}
/** Join failed native retirements without interrupting healthy tasks. */
async function retryFailedRetirements() {
const outcomes = await Promise.allSettled([...slots].filter((slot) => slot.retirementFailed).map((slot) => retire(slot)));
outcomes.push(...await Promise.allSettled(artifactCleanups));
const errors = outcomes.flatMap((outcome) => outcome.status === "rejected" ? [toErrorObject(outcome.reason, "worker retirement retry failed")] : []);
const firstError = errors[0];
if (firstError) throw errors.length === 1 ? firstError : new AggregateError(errors, "Worker retirement retries failed", { cause: firstError });
}
return {
retire,
clearIdle,
idle(slot) {
const idleMs = options.idleTimeoutMs ?? 6e4;
if (idleMs <= 0) return;
if (!warmSlot && now() - lastIdleRetirementAt < WORKER_WARM_WINDOW_MS) warmSlot = slot;
slot.idleTimer = runInContext(() => setTimeoutFn(() => void retire(slot, "idle_timeout").catch(() => void 0), warmSlot === slot ? Math.max(idleMs, WORKER_WARM_WINDOW_MS) : idleMs));
slot.idleTimer.unref();
},
retireIdle(resourceClosures) {
for (const slot of slots) if (!slot.task && !slot.retiring && !slot.retirementFailed && slot.worker && !resourceClosures.get(slot.worker)?.pending) retire(slot, "memory_pressure").catch(() => void 0);
},
retryFailedRetirements,
joinArtifacts: () => Promise.all(artifactCleanups)
};
}
//#endregion
//#region src/infra/worker-task-pool-core.ts
const runInWorkerPoolContext = AsyncLocalStorage.snapshot();
var WorkerTaskError = class extends Error {
constructor(message, code) {
super(message);
this.code = code;
this.name = "WorkerTaskError";
}
};
/** Bounded execution workers; each worker accepts one task at a time. */
var WorkerTaskPoolCore = class {
constructor(options, publicDispatch) {
this.options = options;
this.publicDispatch = publicDispatch;
this.slots = /* @__PURE__ */ new Set();
this.ownedTasks = /* @__PURE__ */ new Set();
this.resourceClosures = /* @__PURE__ */ new WeakMap();
this.ownedSettlement = {
cancel: (task) => this.cancel(task, new WorkerTaskError("worker task closed", "unavailable")),
detach: (task) => {
if (task.slot?.task === task) {
task.slot.task = void 0;
this.activeTasks--;
}
},
retire: (slot) => this.retirement.retire(slot),
release: (task, slot) => {
if (!this.ownedTasks.delete(task)) return;
this.releaseAdmission(task);
task.runInContext = runInWorkerPoolContext;
this.dispatch();
if (slot && !task.owner?.retire && !slot.task && !slot.retiring) this.idle(slot);
}
};
this.completion = {
preparationCleanups: /* @__PURE__ */ new Map(),
releaseAdmission: (task) => this.releaseAdmission(task),
releaseCompute: (permit) => this.computeCapacity.release(permit),
diagnostics: () => ({
worker: this.options.workerUrl.pathname.split("/").at(-1),
...this.publicDispatch?.getSnapshot() ?? this.getSnapshot(),
pendingBytes: this.pendingBytes
})
};
this.queue = [];
this.pendingTasks = 0;
this.pendingBytes = 0;
this.workers = 0;
this.workersCreated = 0;
this.activeTasks = 0;
this.resumeCompute = () => this.dispatch();
this.rotationFailed = false;
this.nextTaskId = 0;
this.retireIdleOnPressure = () => this.retirement.retireIdle(this.resourceClosures);
this.maxWorkers = options.maxWorkers ?? availableParallelism();
this.maxPendingTasks = options.maxPendingTasks ?? 128;
this.maxPendingBytes = options.maxPendingBytes ?? 268435456;
for (const [name, value] of Object.entries({
maxWorkers: this.maxWorkers,
maxPendingTasks: this.maxPendingTasks,
maxPendingBytes: this.maxPendingBytes
})) if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
this.computeCapacity = options.sharedCompute ? getWorkerComputeCapacity() : void 0;
this.retirement = createWorkerTaskPoolRetirement({
slots: this.slots,
options,
runInContext: runInWorkerPoolContext,
dispatch: () => this.dispatch()
});
}
run(input, options) {
if (this.closedError) return Promise.reject(this.closedError);
const inputBytes = options.inputBytes ?? 0;
if (!Number.isSafeInteger(inputBytes) || inputBytes < 0) return Promise.reject(/* @__PURE__ */ new RangeError("inputBytes must be a nonnegative safe integer"));
return this.enqueue(input, options, false, inputBytes).promise;
}
runTask(input, options) {
const task = this.enqueue(input, options, true);
return {
result: task.promise,
close: (closeOptions) => joinOwnedWorkerTask(task, this.ownedSettlement, closeOptions?.retire === true)
};
}
enqueue(input, options, owned = false, inputBytes = options.inputBytes ?? 0) {
const task = {
...createDeferredCore(),
id: ++this.nextTaskId,
runInContext: AsyncLocalStorage.snapshot(),
controller: new AbortController(),
inputConsumed: false,
executionNotified: false,
exchangeSequence: 0,
input,
options: { ...options },
abort: () => this.cancel(task, toErrorObject(options.signal?.reason, "worker task aborted")),
done: false,
admitted: false,
...owned ? { owner: {
closed: false,
retire: false
} } : {},
inputBytes,
enqueuedAt: performance.now(),
transferMs: 0
};
if (owned) this.ownedTasks.add(task);
if (this.closedError || !Number.isSafeInteger(inputBytes) || inputBytes < 0) {
this.finish(task, this.closedError ?? /* @__PURE__ */ new RangeError("inputBytes must be a nonnegative safe integer"));
return task;
}
if (this.pendingTasks >= this.maxPendingTasks || this.pendingBytes + inputBytes > this.maxPendingBytes || this.computeCapacity && !this.computeCapacity.admit(inputBytes)) {
this.finish(task, new WorkerTaskError("worker task capacity reached", "overloaded"));
return task;
}
task.admitted = true;
this.pendingTasks++;
this.pendingBytes += inputBytes;
if (options.timeoutMs !== void 0) this.armTimeout(task, options.timeoutMs);
options.signal?.addEventListener("abort", task.abort, { once: true });
this.queue.push(task);
if (options.signal?.aborted) task.abort();
else this.dispatch();
return task;
}
get isClosed() {
return this.closedError !== void 0;
}
getSnapshot() {
return {
maxWorkers: this.maxWorkers,
workers: this.workers,
workersCreated: this.workersCreated,
activeTasks: this.activeTasks,
pendingTasks: this.pendingTasks
};
}
retryFailedRetirements() {
return this.retirement.retryFailedRetirements();
}
/** Close a retained native resource without cancelling other paths' tasks. */
async closeResources(key) {
await closeWorkerPoolResources(this.slots, this.resourceClosures, key);
}
/** Pause dispatch, settle current work and join native exit before restarting the queue. */
rotate() {
if (this.rotation) return this.rotation;
const completion = createDeferredCore();
this.rotation = completion.promise;
this.rotationFailed = false;
this.computeCapacity?.remove(this.resumeCompute);
const slots = [...this.slots];
const tasks = slots.flatMap((slot) => slot.task ? [slot.task] : []);
Promise.allSettled(tasks.map((task) => task.promise)).then(() => Promise.all(slots.map((slot) => this.retirement.retire(slot, "rotation")))).then(() => this.retirement.joinArtifacts()).then(() => {
this.rotation = void 0;
completion.resolve();
this.dispatch();
}, (error) => {
this.rotation = void 0;
this.rotationFailed = true;
completion.reject(error);
});
return completion.promise;
}
close(error = new WorkerTaskError("worker task pool closed", "unavailable")) {
this.closedError ??= error;
channel("openclaw.memory.critical").unsubscribe(this.retireIdleOnPressure);
this.computeCapacity?.remove(this.resumeCompute);
for (const task of this.queue.splice(0)) this.finish(task, this.closedError);
for (const slot of this.slots) {
if (slot.worker) markWorkerRetirement(slot.worker, "closed");
if (slot.task && !slot.task.owner) this.finish(slot.task, this.closedError, void 0, true);
}
const tasks = [...this.ownedTasks];
const ownedSlots = new Set(tasks.flatMap((task) => task.slot?.task === task ? [task.slot] : []));
const owned = tasks.map((task) => joinOwnedWorkerTask(task, this.ownedSettlement, true));
const unowned = [...this.slots].filter((slot) => !ownedSlots.has(slot));
const closures = [...owned, ...unowned.map((slot) => this.retirement.retire(slot))];
return (tasks.length ? joinOwnedWorkerTasks(closures) : Promise.all(closures)).then(() => joinWorkerTaskPreparationCleanups(this.completion, this.retirement.joinArtifacts()));
}
dispatch() {
if (!this.slots.size) channel("openclaw.memory.critical").unsubscribe(this.retireIdleOnPressure);
if (!this.queue.length || this.closedError || this.rotation || this.rotationFailed) this.computeCapacity?.remove(this.resumeCompute);
while (!this.closedError && !this.rotation && !this.rotationFailed && this.queue.length) {
let slot = [...this.slots].find((entry) => !entry.task && !entry.retiring && !entry.retirementFailed);
if (!slot) {
if (this.slots.size >= this.maxWorkers) {
let contenders = this.queue.length;
for (const occupied of this.slots) if (occupied.task?.exchange && !occupied.task.exchange.sent && !occupied.task.exchange.pressure.signal.aborted && contenders-- > 0) occupied.task.exchange.pressure.abort();
return;
}
}
const nextTask = this.queue[0];
if (this.computeCapacity) {
const permit = this.computeCapacity.acquire(this.resumeCompute, () => {
if (nextTask.exchange && !nextTask.exchange.sent) {
nextTask.runInContext(() => nextTask.exchange?.pressure.abort());
return true;
}
return false;
});
if (!permit) return;
nextTask.computePermit = permit;
}
if (!slot) {
slot = { nativeSections: createWorkerNativeSectionState() };
this.slots.add(slot);
}
this.retirement.clearIdle(slot);
const task = this.queue.shift();
slot.task = task;
this.activeTasks++;
task.slot = slot;
task.startedAt = performance.now();
slot.worker?.ref();
task.runInContext(() => this.start(slot, task));
}
}
createWorker(slot) {
if ((this.options.idleTimeoutMs ?? 6e4) > 0) {
const pressure = channel("openclaw.memory.critical");
pressure.unsubscribe(this.retireIdleOnPressure);
pressure.subscribe(this.retireIdleOnPressure);
}
const worker = runInWorkerPoolContext(() => {
const prepared = this.options.prepareWorker?.();
slot.releaseResources = prepared?.releaseResources;
const temporaryDirectory = prepared?.temporaryDirectory;
if (temporaryDirectory) {
const releaseResources = slot.releaseResources;
slot.releaseResources = async () => {
try {
const { removeTemporaryArtifacts } = await import("./temp-artifact-cleanup-CQWYU68y.mjs");
await removeTemporaryArtifacts(temporaryDirectory, "Worker task");
} finally {
await releaseResources?.();
}
};
}
const workerUrl = this.options.workerUrl;
const workerOptions = {
execArgv: resolveRuntimeWorkerThreadExecArgv(workerUrl),
...this.options.workerOptions,
...prepared?.options
};
if (slot.retiring) throw new WorkerTaskError("worker creation closed during preparation", "unavailable");
return createCpuTrackedWorker(workerUrl, workerOptions);
});
this.workers++;
this.workersCreated++;
attributeWorkerToPool(worker, this);
slot.worker = worker;
worker.on("message", (message) => {
if (receiveWorkerMemoryPort(worker, message)) return;
const task = slot.task;
if (task) task.runInContext(() => this.receive(slot, message));
else this.receive(slot, message);
});
worker.on("error", (error) => this.fail(slot, new WorkerTaskError(String(error), "unavailable")));
worker.on("messageerror", (error) => this.fail(slot, new WorkerTaskError(String(error), "unavailable")));
worker.once("exit", (code) => {
releaseWorkerNativeSectionsOnExit(slot.nativeSections);
this.workers--;
this.fail(slot, new WorkerTaskError(`worker exited with code ${code}`, "unavailable"));
});
return worker;
}
async start(slot, task) {
const taskInput = task.input;
delete task.input;
let input;
task.preparation = createDeferredCore();
try {
try {
input = typeof taskInput === "function" ? await taskInput() : taskInput;
} finally {
task.preparation.resolve();
task.preparation = void 0;
}
} catch (error) {
this.finish(task, toErrorObject(error, "worker task preparation failed"));
return;
}
if (task.done) return;
task.preparedAt = performance.now();
try {
const worker = slot.worker ?? this.createWorker(slot);
const transferList = task.options.transferList?.(input);
if (!task.done) {
const transferStartedAt = performance.now();
worker.postMessage({
input,
taskId: task.id,
interactive: Boolean(task.options.onRequest),
nativeSections: slot.nativeSections.buffer,
sampleMemory: true
}, transferList);
task.transferMs += performance.now() - transferStartedAt;
}
} catch (error) {
this.fail(slot, new WorkerTaskError(String(error), "unavailable"));
}
}
receive(slot, message) {
if (slot.retiring) return;
const task = slot.task;
if (task && isRecord(message) && (message.status === "request" || message.status === "consumed")) {
try {
this.receiveExchange(slot, task, message);
} catch (error) {
this.fail(slot, toErrorObject(error, "worker consumption callback failed"));
}
return;
}
if (!task || !isRecord(message) || (task.options.onRequest || message.taskId !== void 0) && message.taskId !== task.id || message.status !== "ok" && message.status !== "failed") {
this.fail(slot, new WorkerTaskError("invalid worker task response", "unavailable"));
return;
}
const reply = message;
if (reply.status === "failed") {
this.finish(task, new WorkerTaskError(reply.error, "failed"), void 0, Boolean(task.exchange) || Boolean(task.options.onInputConsumed) && !task.inputConsumed);
return;
}
try {
this.options.validateResult?.(reply.value);
} catch (error) {
this.fail(slot, toErrorObject(error, "worker result validation failed"));
return;
}
this.finish(task, void 0, reply.value, Boolean(task.exchange) || Boolean(task.options.onInputConsumed && !task.inputConsumed));
}
armTimeout(task, timeoutMs) {
clearTimeout(task.timer);
task.timer = setTimeout(() => this.cancel(task, new WorkerTaskError("worker task timed out", "timeout")), resolveTimerTimeoutMs(timeoutMs, 6e4));
}
receiveExchange(slot, task, message) {
if (message.taskId !== task.id) {
this.fail(slot, new WorkerTaskError("stale worker exchange", "unavailable"));
return;
}
if (message.status === "consumed") {
if (message.id === 0 && !task.inputConsumed) {
task.inputConsumed = true;
const release = task.options.onInputConsumed;
task.options.onInputConsumed = void 0;
release?.();
} else if (task.exchange?.sent && message.id === task.exchange.id) {
const release = task.exchange.onConsumed;
task.exchange = void 0;
release?.();
} else this.fail(slot, new WorkerTaskError("invalid worker consumption receipt", "unavailable"));
return;
}
if (!task.options.onRequest || task.exchange || !Number.isSafeInteger(message.id) || message.id !== task.exchangeSequence + 1) {
this.fail(slot, new WorkerTaskError("invalid worker exchange", "unavailable"));
return;
}
clearTimeout(task.timer);
const exchange = {
id: ++task.exchangeSequence,
pressure: new AbortController(),
sent: false,
onConsumed: void 0
};
task.exchange = exchange;
this.dispatch();
this.computeCapacity?.requestCheckpoints();
Promise.resolve().then(() => {
if (task.done || slot.task !== task) throw new WorkerTaskError("worker task closed before host dispatch", "unavailable");
return task.options.onRequest(message.value, {
signal: task.controller.signal,
yieldSignal: exchange.pressure.signal
});
}).then((response) => {
if (task.done || slot.task !== task || slot.retiring) {
const release = () => {
try {
task.runInContext(() => response.onConsumed?.());
} catch {}
};
if (this.slots.has(slot)) (slot.completions ??= []).push(release);
else release();
return;
}
exchange.onConsumed = response.onConsumed;
exchange.sent = true;
this.armTimeout(task, response.timeoutMs);
try {
slot.worker.postMessage({
taskId: task.id,
responseId: exchange.id,
input: response.input
}, response.transferList);
} catch (error) {
this.fail(slot, toErrorObject(error, "worker response delivery failed"));
}
}).catch((error) => {
if (!task.done && slot.task === task) this.fail(slot, toErrorObject(error, "worker host exchange failed"));
});
}
cancel(task, error) {
if (task.done) return;
if (task.slot) {
if (task.slot.worker) markWorkerRetirement(task.slot.worker, "cancelled");
if (task.owner) this.finish(task, error, void 0, true);
else this.fail(task.slot, error);
} else {
this.queue.splice(this.queue.indexOf(task), 1);
this.finish(task, error);
}
}
fail(slot, error) {
if (slot.retiring) return;
if (slot.worker) markWorkerRetirement(slot.worker, "failure");
if (slot.task?.owner) {
if (slot.task.done) {
if (!slot.task.owner.retire) closeOwnedWorkerTask(slot.task, this.ownedSettlement, true).catch(() => void 0);
} else this.finish(slot.task, error, void 0, true);
} else if (this.options.restartOnError === false) (this.publicDispatch?.close(error) ?? this.close(error)).catch(() => void 0);
else if (slot.task) this.finish(slot.task, error, void 0, true);
else this.retirement.retire(slot).catch(() => void 0);
}
finish(task, error, value, retire = false) {
if (task.done) return;
task.done = true;
task.runInContext(() => task.controller.abort());
clearTimeout(task.timer);
task.options.signal?.removeEventListener("abort", task.abort);
const complete = createWorkerTaskCompletion(task, this.completion, error, value);
if (task.owner) {
task.owner.complete = complete;
task.owner.retire ||= retire || Boolean(error && task.slot);
if (error) task.reject(error);
else task.resolve(value);
if (error || retire) closeOwnedWorkerTask(task, this.ownedSettlement).catch(() => void 0);
return;
}
const slot = task.slot;
if (slot) {
slot.task = void 0;
this.activeTasks--;
if (retire) {
(slot.completions ??= []).push(complete);
this.retirement.retire(slot, "failure").catch((failure) => {
task.reject(error ? new AggregateError([error, failure], `Worker retirement failed: ${toErrorObject(failure, "worker retirement failed").message}; task failed: ${error.message}`, { cause: failure }) : failure);
});
return;
}
}
complete();
this.dispatch();
if (slot && !slot.task && !slot.retiring) this.idle(slot);
}
releaseAdmission(task) {
if (task.admitted) {
task.admitted = false;
this.pendingTasks--;
this.pendingBytes -= task.inputBytes;
this.computeCapacity?.finish(task.inputBytes);
}
}
idle(slot) {
if (slot.worker && !this.resourceClosures.get(slot.worker)?.pending) slot.worker.unref();
this.retirement.idle(slot);
}
};
//#endregion
//#region src/infra/worker-task-pool.ts
/** Existing SDK surface; task custody remains an internal capability. */
var WorkerTaskPool = class {
constructor(options) {
this.core = new WorkerTaskPoolCore(options, {
close: (error) => this.close(error),
getSnapshot: () => this.getSnapshot()
});
}
run(input, options) {
return this.core.run(input, options);
}
get isClosed() {
return this.core.isClosed;
}
getSnapshot() {
return this.core.getSnapshot();
}
retryFailedRetirements() {
return this.core.retryFailedRetirements();
}
rotate() {
return this.core.rotate();
}
close(error) {
return this.core.close(error);
}
};
//#endregion
export { WorkerTaskPool as t };