comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
1,284 lines • 75 kB
JavaScript
/**
* Background registry for model downloads.
*
* `download_model` used to await the whole transfer before returning anything.
* On a multi-GB checkpoint that left the agent's tool call pending for minutes:
* the turn stayed alive burning tokens with nothing to say, and when the user
* forced a message to break the apparent hang, the pending call was cancelled —
* so the agent reported the download as not done. The stream kept writing to
* disk regardless, which is the worst version of the bug: the file arrives and
* the agent tells you it didn't. (Reported in Discord; issue #290.)
*
* This registry lets the tool hand back a handle instead of blocking forever,
* mirroring the move generation already made from a blocking `run_workflow` to
* `enqueue_workflow` + polling job status (`queue` action:"status").
*/
import { createHash } from "node:crypto";
import { realpath } from "node:fs/promises";
import { extname, basename, dirname, join } from "node:path";
import { downloadModel, liveListingHasEntry, managerJobFilename, localDownloadCacheIdentity, resolveDownloadTarget, shouldDispatchDownloadToManager, verifyLandedModel, verifyManagerVisibility, } from "./model-resolver.js";
import { ModelError } from "../utils/errors.js";
import { readDownloadProgress, listInFlightDownloadProgress, persistDownloadJob, currentDownloadTarget, listPersistedDownloadJobs, findPersistedDownloadJob, removePersistedDownloadJobFor, clearDownloadProgress, persistedRecordsEnabled, PERSIST_OWNER, PERSISTED_INFLIGHT_STALE_MS, } from "./download-progress.js";
import { logger } from "../utils/logger.js";
import { observeStagedPartialAtPath } from "./download-cache.js";
/** How often an in-flight job re-persists its record (liveness heartbeat). Must stay
* comfortably below PERSISTED_INFLIGHT_STALE_MS so a live download is always fresh. */
const HEARTBEAT_MS = 15_000;
/** Cap on heartbeat retries of a transiently-failing TERMINAL persist. Once this
* bounded retry window ends the completed job stops touching the filesystem; its
* last in-flight snapshot remains non-adoptable after the freshness window and is
* reaped only by the long persisted-record retention bound (#761). */
const TERMINAL_PERSIST_MAX_ATTEMPTS = 6; // 6 × 15s = 90s > 60s stale window
// The in-flight registry is indexed under MULTIPLE keys per job so dedup is
// ROUTE-INDEPENDENT: a repeated request finds the in-flight job whether or not the
// Manager↔local route flipped between calls (#420 codex round 2). One Entry object
// is shared by all its keys.
const jobs = new Map();
// Per-DESTINATION-PATH serialization chain (#467 P1-C). Two concurrent jobs for
// the SAME on-disk destination with DIFFERENT auth are (correctly) distinct jobs
// with distinct representations — but they both materialize to that ONE path, and
// each runs post-download work (onComplete) against it. Running them concurrently
// lets one job's writer replace the destination WHILE the other's onComplete reads
// it — so Alice's callback could process Bob's bytes. We serialize by the auth-free
// destination so each job's download+materialize+onComplete sees ITS OWN bytes
// uninterrupted; the final on-disk file is deterministically the last-started job's
// (inherent: one path can hold one file). Keyed by the realpath-collapsed,
// case-normalized LOCAL targetPath, OR a canonical remote:<subfolder>/<name> for
// Manager-dispatched jobs (which write ONE server-side file per subfolder+name).
const destChains = new Map();
/** Canonicalize a LOCAL destination key for the serialization chain. Case-
* insensitive filesystems (Windows always; macOS by default) alias `Checkpoints/m`
* and `checkpoints/m` to ONE physical file, so lower-case the key there to
* serialize those too (#467 P1-C). POSIX (Linux) is case-sensitive — leave exact. */
function normalizeLocalDestKey(key) {
return process.platform === "win32" || process.platform === "darwin"
? key.toLowerCase()
: key;
}
/** realpath the DEEPEST EXISTING ANCESTOR of `p` and re-append the not-yet-created
* tail. `realpath(p)` alone fails (and falls back to the lexical path) whenever any
* descendant is absent — but the writer mkdir's the tree later, so a symlink/
* junction in the EXISTING prefix must still be collapsed (#467 P1-C). Walking up
* to the deepest existing dir collapses that prefix regardless of the missing tail. */
async function realpathDeepestExisting(p) {
const tail = [];
let current = p;
for (;;) {
try {
const real = await realpath(current);
return tail.length ? join(real, ...tail.reverse()) : real;
}
catch (err) {
const code = err?.code;
// Only ABSENCE walks upward. An existing-but-unresolvable ancestor (EIO,
// EACCES, ELOOP, …) must NOT be treated as absent — walking past it would
// reconstruct the un-collapsed lexical ALIAS and defeat serialization (wrong-
// bytes race). Fail CLOSED: propagate so the job errors rather than risk it
// (#467). A transient error here is far rarer than the corruption it guards.
if (code !== "ENOENT" && code !== "ENOTDIR")
throw err;
const parent = dirname(current);
if (parent === current)
return p; // reached the root without resolving — give up
tail.push(basename(current));
current = parent;
}
}
}
/** The LOCAL serialization key: collapse symlinks/junctions in the destination path
* (via the deepest existing ancestor), then case-normalize — so two aliased
* subfolders resolving to ONE physical file share a chain (#467 P1-C). */
async function localSerializeKey(targetPath) {
const real = await realpathDeepestExisting(targetPath);
return normalizeLocalDestKey(real);
}
/** The REMOTE serialization key. The Manager writes ONE server-side file per
* (subfolder, name); derive the name the way the resolve/Manager path does
* (URL pathname basename when no explicit filename) and normalize separators so
* `a//b` == `a/b` and query variants collapse. The remote host's case-sensitivity
* is UNKNOWN here, so lower-case unconditionally — over-serializing is safe, under-
* serializing (concurrent same-file writes) is not (#467 P1-C). */
function remoteSerializeKey(url, targetSubfolder, filename) {
let name = filename;
if (!name) {
try {
name = basename(new URL(url).pathname);
}
catch {
name = url.split(/[/?#]/).filter(Boolean).pop();
}
}
// Mirror the Manager writer's trim() then separator-normalize so ` loras/sub `,
// `loras/sub`, and `loras//sub` all canonicalize to one key (#467 P1-C).
const sub = String(targetSubfolder ?? "")
.trim()
.split(/[/\\]+/)
.map((s) => s.trim())
.filter(Boolean)
.join("/");
return `remote:${sub}/${(name || "model").trim()}`.toLowerCase();
}
/**
* Read the pre-download listing baseline as a TRI-STATE (#1374 review, P1-1).
*
* Three distinct things used to arrive as `undefined` and be treated alike:
* the caller named no file, the listing call threw, and the server answered
* "cannot say". None of them is "the server did not list it before", and reading
* them as that is what lets a pre-existing file be credited to a dispatch that
* fetched nothing. Only a real, answered `false` is a negative baseline.
*
* Never throws — a baseline is a nicety and must not fail a download.
*/
async function captureListingBaseline(targetSubfolder, name) {
if (!name)
return "unknown";
try {
const has = await liveListingHasEntry(targetSubfolder, name);
// liveListingHasEntry ITSELF returns undefined for "could not be asked", and
// that is the state this whole helper exists to keep distinct.
return has === undefined ? "unknown" : has;
}
catch {
return "unknown";
}
}
/**
* Point `key` at `entry`, ENTRY-SCOPED (#420 codex round 3, rule 2/3): never clobber
* a row currently owned by a DIFFERENT, still-in-flight writer — that would orphan a
* live download's index. Records the key on the entry so it can be retired later.
*/
function registerKey(entry, key) {
const cur = jobs.get(key);
if (cur && cur !== entry && cur.job.status === "downloading")
return;
if (!entry.keys.includes(key))
entry.keys.push(key);
jobs.set(key, entry);
}
/**
* Retire a superseded (done/error) entry, ENTRY-SCOPED (#420 codex round 3, rule 2):
* only delete an index row that STILL points at THIS entry, so retiring an older job
* can never delete a key that has since been reassigned to a newer/live entry.
*/
function retireEntry(entry) {
for (const key of entry.keys) {
if (jobs.get(key) === entry)
jobs.delete(key);
}
}
/** The tray keys rows on a hash of the SOURCE URL; match it so both agree. */
export function downloadIdFor(url) {
return createHash("sha256").update(url).digest("hex").slice(0, 16);
}
/**
* The download's DISTINCT public id — a hash of the given identity string. Callers
* pass the canonical resolved on-disk `targetPath` PLUS an auth discriminator
* (#467 P1-A), so identity is the DESTINATION *and representation*: two requests
* that resolve to the SAME file with the SAME auth are one job/one writer (even
* from different URLs), but the SAME file with DIFFERENT auth are DIFFERENT
* downloads (a different representation) and must not dedup. Requests to different
* destinations are separately pollable via download_model action:"status".
*/
export function downloadJobIdFor(identity) {
return createHash("sha256").update(identity).digest("hex").slice(0, 16);
}
/**
* ROUTE-INDEPENDENT request key: a canonical, collision-safe JSON-encoded tuple of
* {url, trimmed subfolder, filename}. Derived ONLY from the request inputs — never
* from the resolved destination or the chosen route — so a repeated call for the
* SAME request adopts the in-flight job regardless of a Manager↔local reachability
* flip between calls (#420 codex round 2). Every job is indexed under this key; a
* locally-resolvable job is ALSO indexed under its destination key (below) so the
* "two different URLs → one local destination → one writer" dedup still holds.
*/
/** A stable per-request discriminator for the caller's auth/representation. The
* JOB layer dedups BEFORE the header-aware cache layer is reached, and it is
* otherwise auth-blind — so without this, two concurrent same-URL+same-dest
* calls with DIFFERENT bearer/custom headers would adopt the FIRST job and the
* second caller would get the first's bytes AND resume callback (#467 P1-A). The
* `auth` param is the per-caller differentiator (config-global tokens are
* identical across concurrent calls, so they need not enter the key). Two auth
* encodings that transmit the SAME headers may still be split here — harmless:
* they then coalesce correctly at the cache layer (same representation). */
function authIdentity(auth) {
return auth ? createHash("sha256").update(JSON.stringify(auth)).digest("hex").slice(0, 12) : "";
}
function requestDownloadKey(url, targetSubfolder, filename, auth, modelRoot) {
const parts = [
url,
String(targetSubfolder ?? "").trim(),
filename ?? null,
authIdentity(auth),
];
// Omit when unset so existing in-flight jobs (no model_root) keep the same key.
const root = String(modelRoot ?? "").trim();
if (root)
parts.push(root);
return createHash("sha256").update(JSON.stringify(parts)).digest("hex").slice(0, 16);
}
/**
* Start a download, or adopt one already running for the same on-disk destination.
*
* The adoption case matters: the visible symptom of the old bug was "the agent
* looks stuck", and the natural user response is to ask again. Without this,
* the second ask starts a SECOND stream onto the same target path — two writers,
* one file. Returning the in-flight job makes a repeated request harmless.
*
* Async because the destination is resolved by the SAME code the write uses
* (resolveDownloadTarget), so the job is keyed by the exact `targetPath` the file
* lands at — any two inputs resolving to one file are one job — and an INVALID
* input (blank / path-ful filename, escaping subfolder) is REJECTED here, up
* front, exactly as the download itself would reject it (never basename'd into a
* collision with a valid request). REMOTE mode still resolves a canonical
* server-side targetPath for identity even though the bytes are fetched by the
* Manager, so duplicate remote dispatches to one destination also dedupe.
*/
export async function startDownloadJob(url, targetSubfolder, filename, auth,
/** Post-download work (sidecars, type checks). Its lines land on `job.notes`
* so they reach the user even when the download outlives the tool call. */
onComplete,
/** Optional call-scoped route decision supplied by apply_manifest. When present,
* it must drive both the identity key and the writer, just like the decision
* computed here for ordinary download_model calls. */
dispatchToManagerOverride,
/** Optional explicit model-root directory already known from list_paths (#2499). */
modelRoot) {
// Identity depends on mode. LOCAL: resolve the canonical on-disk destination
// with the SAME resolver the write uses (throws on an invalid filename/subfolder
// — surfaced immediately, matching downloadModel's own rejection — and keys by
// the exact targetPath so identity is the destination, not the URL). REMOTE:
// there is NO local filesystem — downloadModel short-circuits to the Manager
// dispatch, so resolving a local models dir would wrongly THROW (COMFYUI_PATH
// unset). Key by a canonical remote identity instead and let downloadModel take
// the Manager path. shouldDispatchDownloadToManager() also covers the #420
// reconnect case: a nominally-local session whose effective base was lost still
// routes through the connected Manager (and must NOT try to resolve a local
// targetPath, which would throw), keying by the same canonical remote identity.
//
// CRITICAL: evaluate the route EXACTLY ONCE and thread it into downloadModel
// below. The predicate awaits live /system_stats and reads mutable base config,
// so a reconnect/reachability flip between two evaluations would split the job —
// Manager-key + local-writer (or a duplicate job) for one request (#420 codex
// round 1). One decision keys the identity AND drives the writer.
const dispatchToManager = dispatchToManagerOverride ?? (await shouldDispatchDownloadToManager());
// DEDUP INDEX (route-independent) vs WRITER ROUTE (the single decision above) are
// deliberately separated (#420 codex round 2). The in-flight lookup/registration
// is keyed by the REQUEST (url+subfolder+filename), which NEVER changes with the
// route — so two separate calls for one request with a Manager↔local flip between
// them still resolve to ONE job (no same-file double-write). When the request is
// locally resolvable we ALSO index the destination-path key, preserving the
// "two different URLs → same local destination → one writer" dedup (WS-4). An
// invalid filename/subfolder is REJECTED here (by resolveDownloadTarget), up
// front, exactly as the write would reject it.
const reqKey = requestDownloadKey(url, targetSubfolder, filename, auth, modelRoot);
let destKey;
// The key jobs SERIALIZE on (#467 P1-C) — the physical destination, AUTH-FREE and
// normalized for case-insensitive filesystems, so two different-auth (or
// different-cased) requests to the same file run one-at-a-time and each callback
// sees its own bytes. LOCAL: the resolved on-disk targetPath. REMOTE: a canonical
// "remote:<subfolder>/<name>" — the Manager writes ONE server-side file per
// (subfolder, name), so overlapping different-auth dispatches to it are serialized
// too (the local callback race is inherently local, but this keeps the server
// write single-writer and the last-started result deterministic).
let serializeKey;
/** Did the live server ALREADY list this exact entry before we started? Filled in
* by `captureListingBaseline` inside `settled` — NOT here. See #1374 review P1-2
* at the capture site for why the position matters. */
let listedBefore = "unknown";
/** The entry the baseline is about: the RESOLVED local filename on the local
* route, the requested name on the Manager route. `undefined` on the Manager
* route when the caller named no file — the Manager decides the name from the
* URL and only reports it in the dispatch descriptor, so guessing one here
* would baseline the WRONG entry, which is worse than having no baseline. */
let baselineName;
if (!dispatchToManager) {
const target = await resolveDownloadTarget(url, targetSubfolder, filename, modelRoot);
baselineName = target.filename;
// Fold auth into the destination key too (#467 P1-A): two concurrent calls to
// the SAME on-disk destination with DIFFERENT auth are DIFFERENT downloads
// (different representations) and must NOT dedup to one writer/one job.
destKey = downloadJobIdFor(`${target.targetPath}\n${authIdentity(auth)}`);
serializeKey = await localSerializeKey(target.targetPath);
}
else {
// #1374 — BASELINE THE MANAGER ROUTE TOO. The post-dispatch check below asks
// the connected server whether it now lists the file; without a baseline, a
// pre-existing file of the same name reads as a successful landing, which is
// the exact trap `listedBefore` was added for on the local path (#369). The
// Manager route resolves no local target, so the name asked for is the name
// to ask about — that is also what `managerJobFilename` will report, so the
// baseline and the check are about the same entry.
baselineName = filename;
serializeKey = remoteSerializeKey(url, targetSubfolder, filename);
}
// The PUBLIC id (download_model action:"status" handle): the destination key when we have one
// (so distinct destinations are separately pollable), else the request key.
const id = destKey ?? reqKey;
// This call's keys: the route-independent request key, plus the destination key
// when locally resolvable (two-URLs-one-dest dedup).
const lookupKeys = destKey ? [reqKey, destKey] : [reqKey];
const trayId = downloadIdFor(url);
const downloadTarget = currentDownloadTarget();
// ADOPT ONLY AN IN-FLIGHT ENTRY (#420 codex round 3, rule 3): a FINISHED entry
// under any key is treated as absent — it must never shadow a currently-downloading
// writer reachable under another key, nor cause a retire that overwrites a live row.
// Scan every key and prefer an in-flight match.
let adopted;
for (const k of lookupKeys) {
const e = jobs.get(k);
if (e && e.job.status === "downloading") {
adopted = e;
break;
}
}
if (adopted) {
// Re-index THIS call's keys onto the adopted entry (#420 codex round 3, rule 1):
// when URL B adopts URL A's in-flight job by destination, B's request key must
// now point at the same entry too — otherwise a later repeat of B (especially
// after a local→Manager flip that drops the destination key) would miss it and
// start a second writer onto one file. Entry-scoped, so it can't steal a live row.
for (const k of lookupKeys)
registerKey(adopted, k);
logger.info(`Download already in flight, adopting it: ${adopted.job.id}`, {
url,
target_subfolder: targetSubfolder,
filename,
});
return adopted;
}
// #529 double-writer guard: no in-flight job in THIS process, but ANOTHER session may
// already be running this EXACT download (same public id) — e.g. a reconnect reissued
// the same URL/destination. Adopt its persisted in-flight record instead of starting a
// SECOND physical writer for one file. Only a FRESH in-flight record from a DIFFERENT
// owner counts (the scan below filters stale via the heartbeat window; and a terminal
// record — done/cancelled — is NOT "downloading", so a finished/failed prior download
// correctly falls through to a fresh re-download). This
// process's own live jobs were already checked above. The adopted view is READ-ONLY (we
// hold no handle on the other session's writer): it is NOT registered in this registry,
// runs no writer/heartbeat/AbortController, and its `settled` resolves immediately — the
// tool then reports it as in-flight and the caller polls download_model action:"status" (which reads
// the live persisted record) for the resolved state.
//
// SCOPE: the persisted-record scan is still BEST-EFFORT and covers the reported case — a
// SEQUENTIAL reconnect that reissues after another session published its record. The
// physical cache writer now has a separate O_EXCL claim on its deterministic shared
// `.partial`; two DISTINCT processes that pass this read-before-start scan cannot both
// mutate that staged identity. The loser receives a refusal/retry error before it can
// observe, truncate, append, or rename the shared partial. Requests with different cache
// identities may still use independent temporary files, and each remains payload-validated
// before its atomic destination rename (#467/#473).
// Scan ROUTE-INDEPENDENTLY (#420): match a foreign fresh in-flight record by the public
// `id` OR the route-independent request key `reqKey`, so a reconnect whose route flipped
// local↔Manager (which changes the id from a destination hash to a request hash) still
// finds the in-flight record instead of starting a second writer. Require the SAME source
// URL (trayId): a foreign live download of a DIFFERENT url to the same dest+auth (same
// id, different trayId) is a distinct logical download whose bytes may differ — adopting
// it would report the WRONG job, so it is declined (we proceed with our own request). A
// reqKey match already implies the same url (reqKey embeds it), so trayId is consistent.
const nowForAdopt = Date.now();
const foreign = listPersistedDownloadJobs().find((rec) => rec.owner !== PERSIST_OWNER &&
rec.status === "downloading" &&
nowForAdopt - (rec.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS &&
rec.trayId === trayId &&
rec.target === downloadTarget &&
(rec.id === id || (rec.req_key !== undefined && rec.req_key === reqKey)) &&
// A FRESH HEARTBEAT IS NOT A LIVE WRITER. The heartbeat is written every
// 15s and only goes stale after 60, so a process that died a moment ago
// leaves a record that still looks current for up to a minute. Adopting it
// hands the caller a job nobody is running: the tool reports "in flight"
// and the poll never settles, which is worse than starting a second
// writer — the outcome adoption exists to avoid — because nothing
// downloads at all.
//
// PROVEN gone only (ESRCH). `undefined` means the probe could not tell —
// no pid on a pre-#858 record, or the probe itself failed — and refusing
// to adopt on that would treat "could not determine" as "dead", which is
// this codebase's own recurring defect (#796) pointed the other way: it
// would start a SECOND writer for a download that is genuinely running.
writerProcessGone(rec) !== true);
if (foreign) {
logger.info(`Adopting a cross-session in-flight download (no second writer): ${foreign.id}`, {
url,
target_subfolder: targetSubfolder,
filename,
});
return {
job: jobFromPersisted(foreign),
settled: Promise.resolve(),
keys: [],
controller: new AbortController(),
};
}
// No in-flight match. Retire any superseded (done/error) entries shadowing our keys
// — entry-scoped, so we only clear rows still pointing at that finished entry and
// never delete a row now owned by a different, live writer (rule 2).
const retired = new Set();
for (const k of lookupKeys) {
const e = jobs.get(k);
if (e && !retired.has(e)) {
retireEntry(e);
retired.add(e);
}
}
const job = {
id,
trayId,
target: downloadTarget,
url,
target_subfolder: targetSubfolder,
filename,
status: "downloading",
started_at: Date.now(),
destKey: serializeKey,
reqKey,
viaManager: dispatchToManager,
};
// Per-download abort handle (#515). The signal is threaded through downloadModel
// into the fetch + stream pipeline, so download_model action:"cancel" aborts exactly THIS job.
const controller = new AbortController();
// The liveness heartbeat timer (assigned below, after the settled closure). Hoisted
// here so the settled closure's finally can clear it once the terminal state is durably
// persisted. Runs async — assigned before it ever fires or the finally runs.
let heartbeat;
// Serialize behind any in-flight job writing the SAME destination (#467 P1-C) so
// this job's download+materialize+onComplete run without a concurrent different-
// auth writer swapping the destination out from under its callback.
const priorSameDest = destChains.get(serializeKey);
// The promise is stored, never left dangling — an unhandled rejection here
// would take down the process on a simple 404.
// The physical download reports its resume decision straight onto THIS job — no
// shared keyed map, so it can never be misattributed to another job (#467).
// Commit the DONE state. Invoked SYNCHRONOUSLY the instant the file is renamed into
// its destination (via the onLanded callback below) — before any later await — so
// there is NO window where the file exists at the destination yet the job still reads
// "downloading" (which a cancel would otherwise flip to "cancelled"). Idempotent and
// only advances a still-"downloading" job, so it never overrides a real terminal
// state. Reads status as the full union to dodge the closure's "downloading" narrowing.
const commitDone = (landedPath) => {
if (job.status !== "downloading")
return;
job.path = landedPath;
job.status = "done";
job.finished_at = Date.now();
// A landing is not yet a verified placement. Mark it PENDING in the SAME
// synchronous step that publishes "done" (and in the same persisted record), so
// no reader — this session's tool call inside its grace window, or another
// session reading the record — can ever see a completed download with no
// verification field and render it as confirmed success (#369, codex gate).
//
// #1374 review, P1-3 — THE MANAGER ROUTE GETS THIS TOO. Its check runs a beat
// after this commit, so `done` with no field was a real, reachable window; a
// reader landing in it could not tell "not checked yet" from "pre-fix record"
// and the tool re-probed the server by hand to cover for it. With the field
// always populated, the record is the single source of the verdict.
job.live_visible = "pending";
// #1545 — this return value was DISCARDED, and it is the one that says
// whether any OTHER process can see the job finish. `persistDownloadJob`
// returns false when the atomic replace loses to a reader holding the record
// open, and it deliberately keeps the PRIOR complete record rather than
// risking a torn write — so the durable answer stays "downloading" until the
// ~15s heartbeat retries.
//
// That is the whole of #1545: the completion event is raised from THIS
// process's memory, while `download_model action:"status"` reads the record
// from the spawned stdio child. Silently dropping the failure is what let
// those two disagree for seconds with nothing to explain it.
//
// One immediate re-attempt (the call has its own backing-off retries), then
// say so. The heartbeat still heals it; the log is what makes a stale read
// explicable instead of mysterious.
if (!persistJobRecord(job) && !persistJobRecord(job)) {
logger.warn(`[download] job ${job.id} finished but its record could not be published ` +
`(${job.filename}). Until the next heartbeat, download_model action:"status" ` +
`may still report "downloading" for it — the file itself has landed at ` +
`${landedPath}.`);
}
};
const settled = (async () => {
// Wait for the prior same-destination job to fully finish (never fail THIS job
// because that one errored — swallow its result).
if (priorSameDest)
await priorSameDest.catch(() => undefined);
// #1374 review, P1-2 — THE BASELINE IS TAKEN HERE, NOT BEFORE THE WAIT.
//
// It used to be read at job-construction time, which is BEFORE this job takes
// its turn in the same-destination chain (#467 P1-C). A second dispatch to the
// same (subfolder, name) therefore baselined a directory the FIRST job had not
// written to yet, sat in the queue while that job landed the file, and then
// read its own post-check as "it appeared → this dispatch landed it" — the
// #369 misattribution, manufactured by the serialization it was queued behind.
// Rejected second dispatches are exactly the case #1374 is about, so this
// window was the bug's own shape.
//
// Taken immediately before `downloadModel`, with nothing but this job's own
// transfer between the two observations.
listedBefore = await captureListingBaseline(targetSubfolder, baselineName);
try {
// A cancel that arrived before (or while waiting for) our turn makes downloadModel
// throw at its up-front abort guard → the catch records the cancellation. So there
// is no separate pre-start branch: the download either runs to a materialized file
// (done) or is aborted before the file lands (cancelled).
const path = await downloadModel(url, targetSubfolder, filename, auth, dispatchToManager, (d) => {
job.resume = d;
}, controller.signal, (progressId) => {
// Record the id the tray rows are ACTUALLY written under (post-auth/HF
// rewrite) as job.progressId — WITHOUT touching job.trayId, which must stay
// the stable original-URL hash so URL reconnect adoption (#529) still
// resolves. download_model action:"status" byte display and cancel cleanup key on
// progressId (falling back to trayId); it only differs for query-auth /
// mirror-rewritten URLs. Persist so a reconnecting session reads live bytes.
if (progressId && progressId !== job.progressId) {
job.progressId = progressId;
persistJobRecord(job);
}
},
// onLanded: fires the moment the local file is renamed into place → commit done
// synchronously, closing the rename→return window.
commitDone, (route) => {
// A redirect may move between a direct bypass and a proxy. Keep the
// conservative answer: once any hop used the proxy, status says proxied.
const effective = route === "proxied" ? "proxied" : (job.downloadRoute ?? "direct");
if (effective !== job.downloadRoute) {
job.downloadRoute = effective;
persistJobRecord(job);
}
}, (partialPath) => {
// The writer has already resolved the effective URL, auth headers, and
// cloud principal here. Persist only its resulting path, never secrets;
// status must inspect this exact identity rather than approximate it
// from the redacted original URL.
if (partialPath && partialPath !== job.partialPath) {
job.partialPath = partialPath;
// The callback's path comes from the physical writer. Recompute the
// proof through that same identity builder only to verify the callback
// and retain a non-secret recovery witness. A mismatch is fail-closed:
// status may still show the exact path, but boot recovery cannot guess.
try {
const identity = localDownloadCacheIdentity(url, auth);
if (identity.partialPath === partialPath) {
job.partialIdentity = identity.partialIdentity;
}
else {
job.partialIdentity = undefined;
logger.warn(`[download] writer partial identity mismatch for ${job.id}; ` +
`automatic recovery is disabled for this record.`);
}
}
catch {
job.partialIdentity = undefined;
}
persistJobRecord(job);
}
}, modelRoot);
// Local paths already committed done via onLanded at the rename. The remote Manager
// dispatch has no local rename, so commit done here when it returns (dispatch
// accepted — the viaManager flag marks it as unverified in the tools). Idempotent.
commitDone(path);
// VERIFY, don't assume (#369). A LOCAL download is reported with the path we
// confirmed on disk, plus whether the connected ComfyUI actually reads from
// there. This runs INSIDE `settled`, so a small download that finishes within
// download_model's grace window is already verified when the tool answers.
// Never allowed to demote a completed download: verifyLandedModel never throws
// and the catch below keeps `done`.
if (!dispatchToManager && job.status === "done") {
try {
const verdict = await verifyLandedModel(path, targetSubfolder, {
// verifyLandedModel's own `visible` verdict is gated on the
// DESTINATION being one the running server named, not on this
// baseline — it uses it only to word an already-inconclusive answer,
// and already words `undefined` as "could not be checked". So the
// tri-state's third state maps onto the `undefined` it expects.
listedBefore: listedBefore === "unknown" ? undefined : listedBefore,
});
if (verdict.verifiedPath)
job.path = verdict.verifiedPath;
job.verified_root = verdict.verifiedAgainstRoot;
// No verifiedPath means the on-disk stat did NOT succeed — the file was
// removed or became unreadable between the rename and this check. Record
// that so no renderer claims "verified on disk" for the retained path.
job.disk_verified = !!verdict.verifiedPath;
job.live_visible = verdict.liveVisible;
job.verify_note = verdict.note;
}
catch (err) {
// Verification is a REPORT, never a gate: the bytes are on disk either
// way. Record that we could not confirm placement and carry on to the
// post-download hook — swallowing this must not skip onComplete. The
// on-disk check did not complete either, so nothing may claim it did.
job.disk_verified = false;
job.live_visible = "unknown";
job.verify_note = `Placement could not be verified: ${err instanceof Error ? err.message : String(err)}`;
}
persistJobRecord(job);
}
// #1374 — ASK, DON'T ASSUME, ON THE MANAGER ROUTE EITHER.
//
// A Manager-dispatched download settles "done" when the Manager QUEUE
// DRAINS, and Manager increments its done counter for a REJECTED item
// exactly as for a fetched one — the v2 status endpoint carries only
// aggregate counts, with no per-item result. So a reporter whose Manager
// refused the fetch outright (security_level / network_mode=personal_cloud)
// got a job that said `done` for a 13.2 GB file that a filesystem-wide
// search could not find. Everything downstream was honest about this in
// PROSE and nothing ever looked.
//
// `verifyManagerVisibility` (#1086) is the look, and it already exists: it
// asks the CONNECTED server whether it now lists the entry, which is
// answerable remotely where `verifyLandedModel`'s on-disk stat is not. This
// runs the same shape as the local branch above, in the same place, so a
// download that settles inside download_model's grace window is already
// checked when the tool answers — and `action:"status"` reads the verdict
// off the record rather than reprinting a static caveat.
//
// WHAT THIS MUST NOT BECOME. "not-listed" is NOT failure and the verdict is
// deliberately not allowed to demote `status`: a Manager dispatch returns on
// ACCEPTANCE, so a large file is still arriving for minutes afterwards, and
// "not there yet" is indistinguishable from "landed where the server cannot
// read" from here. What changes is that the report now carries an OBSERVED
// answer where one is obtainable, instead of a caveat that was true of every
// outcome and therefore distinguished none of them.
if (dispatchToManager && job.status === "done") {
try {
const verdict = await verifyManagerVisibility(targetSubfolder, managerJobFilename(job), { listedBefore });
// Mapped onto the SAME three-valued field the local path writes, so
// every renderer keeps one vocabulary. "not-listed" is the Manager
// route's spelling of "the connected server does not list this".
job.live_visible =
verdict.visibility === "visible"
? "visible"
: verdict.visibility === "not-listed"
? "not-visible"
: "unknown";
job.verify_note = verdict.note;
}
catch (err) {
// verifyManagerVisibility documents that it never throws; this is the
// belt for that brace, and it fails to UNKNOWN — never to a verdict.
job.live_visible = "unknown";
job.verify_note = `Placement could not be verified: ${err instanceof Error ? err.message : String(err)}`;
}
persistJobRecord(job);
}
if (onComplete) {
// Post-processing must not turn a landed file into a failed download —
// the bytes are on disk either way, and reporting "error" here would
// reproduce the very bug this registry exists to fix.
try {
job.notes = await onComplete(path);
}
catch (err) {
job.notes = [
`(post-download step failed: ${err instanceof Error ? err.message : String(err)} — the file itself downloaded fine)`,
];
}
}
}
catch (err) {
// If the file ALREADY landed (onLanded committed done) a later throw — e.g. an LRU
// eviction hiccup after the rename — must NOT override a real completed download.
if (job.status === "done") {
// keep done
}
else if (controller.signal.aborted) {
// An abort that stopped the transfer BEFORE the file landed surfaces here as a
// rejected fetch/pipeline (or a guard throw) — a clean cancellation, never an
// "error" and never a false success.
finalizeCancelled(job);
// …but if the cancellation cleanup threw a recovery-critical ModelError (e.g. the
// Windows backup of the PREVIOUS destination file could not be restored, so it's
// preserved under a random .bak path), surface that message on the job so
// download_model action:"status" shows the recoverable path instead of masking it behind a plain
// "cancelled, resumable partial". (Ordinary cancellation throws an AbortError,
// which is NOT a ModelError, so it never sets this.)
if (err instanceof ModelError)
job.error = err.message;
}
else {
job.status = "error";
job.error = err instanceof Error ? err.message : String(err);
job.finished_at = Date.now();
}
}
finally {
// Persist the terminal state so a reconnecting session sees the resolved
// outcome (#529) instead of a forever-"downloading" record. If this atomic
// replace transiently fails (rare), the heartbeat below KEEPS retrying until the
// terminal state is durable — so a done/cancelled job can't linger as a
// fresh-and-adoptable "downloading" record. The freshness window excludes it from
// adoption immediately; long retention later bounds the persisted record. Only once
// the terminal record is durable does the heartbeat stop.
if (persistJobRecord(job) && heartbeat)
clearInterval(heartbeat);
}
})();
// Make THIS job the tail of its destination's serialization chain, and prune the
// chain entry once it's the last one to settle (bounded map).
destChains.set(serializeKey, settled);
void settled.finally(() => {
if (destChains.get(serializeKey) === settled)
destChains.delete(serializeKey);
});
// Index under EVERY key (deduped) so any of them adopts this one writer. Uses the
// entry-scoped registerKey guard so a fresh registration can never overwrite a row
// still owned by a different, live writer (rule 3).
const entry = { job, settled, keys: [], controller };
for (const k of new Set(lookupKeys))
registerKey(entry, k);
// Persist the (downloading) record so a session that reconnects while this is in
// flight can still resolve/adopt it by id or URL/destination (#529).
persistJobRecord(job);
// Liveness heartbeat: while in flight, refresh the persisted record's `updated`
// stamp so ANOTHER session can distinguish this LIVE download from a crashed one
// (its heartbeat stops) when deciding whether to preserve a shared tray row. Cheap
// (a small write every 15s, only under the panel where persistence is active) and
// unref'd so it never keeps the process alive.
// Only run the heartbeat when the persisted store is actually active (panel). Without a
// channel dir there is nothing to persist and no reader to serve — an interval there
// would just leak, retrying a no-op forever (persistJobRecord returns false, so the
// terminal cleanup below would never clear it). Plain non-panel downloads install none.
if (persistedRecordsEnabled()) {
let terminalPersistAttempts = 0;
heartbeat = setInterval(() => {
// In-flight: refresh liveness so another session can tell this live download from a
// crashed one. Terminal (only reached when the settled finally's terminal persist
// transiently FAILED to atomically replace): keep retrying until it's durable, then
// stop — so a done/cancelled job never lingers as a fresh, adoptable "downloading".
const durable = persistJobRecord(job);
if (job.status !== "downloading") {
// Stop the interval once the terminal state is durable, OR the retry budget is
// spent (after which the snapshot is non-adoptable), OR persistence went inactive
// — never let a completed job's interval do filesystem work forever.
if ((durable ||
++terminalPersistAttempts >= TERMINAL_PERSIST_MAX_ATTEMPTS ||
!persistedRecordsEnabled()) &&
heartbeat) {
clearInterval(heartbeat);
}
}
}, HEARTBEAT_MS);
if (typeof heartbeat.unref === "function")
heartbeat.unref();
entry.heartbeat = heartbeat;
}
return entry;
}
/** Mark a job cancelled: never a false-complete, and drop its tray row. The
* resumable .partial is deliberately LEFT on disk (a later download resumes it). */
function finalizeCancelled(job) {
job.status = "cancelled";
if (job.finished_at === undefined)
job.finished_at = Date.now();
// Remove the panel tray row so a cancelled download doesn't linger — but clear it
// ONLY when this job exclusively owns that physical row. Two subtleties:
// - A job with no progressId never wrote a row (the writer reports progressId
// BEFORE the first row), so there is nothing to clear.
// - Same-URL siblings can SHARE one physical stream/row: a coalesced consumer (same
// URL+auth, different destination) or a serialized same-auth sibling observes the
// OWNER's progressId. Clearing it while ANOTHER in-flight job still uses it would
// wipe that active job's live display. So skip the clear when any other in-flight
// job shares this progressId; the last one out clears it. (trayId is deliberately
// NOT a fallback — it's the original-URL hash siblings share even more broadly.)
if (!job.progressId)
return;
const sharedByLocal = [...new Set(jobs.values())].some((e) => e.job !== job && e.job.status === "downloading" && e.job.progressId === job.progressId);
// Also honor a live sibling in ANOTHER session (#529): a persisted in-flight record
// written by a DIFFERENT session (owner ≠ this process's PERSIST_OWNER) that shares
// this progressId is another session's active download of the same URL — its tray row
// must not be wiped. Comparing by OWNER (not id) catches the case two sessions run the
// SAME logical download (identical deterministic id): they persist distinct
// owner-scoped files, so the other session's record is still seen here. This process's
// OWN persisted records (same owner) are excluded — they always correspond to an
// in-memory job already covered by the local scan above. If any live sibling exists in
// either store, leave the row for the last one out (or the orchestrator's dead-writer
// prune) to clear.
const now = Date.now();
const sharedByPersisted = listPersistedDownloadJobs().some((rec) => rec.owner !== PERSIST_OWNER &&
rec.status === "downloading" &&
rec.progressId === job.progressId &&
// Only a FRESH record counts as a live sibling — a crashed session's heartbeat
// stops, so its record goes stale and must NOT permanently suppress this sole
// job's cleanup (its tray row would otherwise linger). Bounded by the heartbeat
// interval so a genuinely-live foreign download is always seen as fresh.
now - (rec.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS);
if (!sharedByLocal && !sharedByPersisted)
clearDownloadProgress(job.progressId);
}
/** Persist a job record to the cross-session store (progress dir), redacting the
* URL. No-op outside the panel. Keeps the persisted copy in step with the job.
* Returns whether the record was DURABLY replaced (so the heartbeat can retry a
* transiently-failing terminal persist). */
function persistJobRecord(job, owner = PERSIST_OWNER) {
return persistDownloadJob({
id: job.id,
trayId: job.trayId,
progressId: job.progressId,
partialPath: job.partialPath,
partial_identity: job.partialIdentity,
target: job.target,
url: job.url,
target_subfolder: job.target_subfolder,
filename: job.filename,
status: job.status,
path: job.path,
error: job.error,
started_at: job.started_at,
finished_at: job.finished_at,
notes: job.notes,
dest_key: job.destKey,
req_key: job.reqKey,
via_manager: job.viaManager,
download_route: job.downloadRoute,
resume: job.resume,
live_visible: job.live_visible,
verify_note: job.verify_note,
disk_verified: job.disk_verified,
verified_root: job.verified_root,
reclaimed_dead: job.reclaimedDead,
}, owner);
}
function boundedDownloadJobError(value) {
if (typeof value !== "string")
return undefined;
const compact = value.replace(/\s+/g, " ").trim();
return compact ? compact.slice(0, 400) : undefined;
}
/** Turn a stale/frozen in-flight record into the terminal state the tray is reporting. */
export function reconcileStalledDownloadRecord(job, error) {
if (job.status !== "downloading")
return true;
const wasPersisted = job.updated !== undefined;
job.status = "error";
job.error = boundedDownloadJobError(error);
job.finished_at = Date.now();
job.updated = job.finished_at;
// A reconstructed foreign record must be terminalized in its ORIGINAL owner file;
// writing a fresh current-owner snapshot leaves the original fresh downloading row
// authoritative to status. Legacy persisted records without an owner cannot be safely
// reconciled, so callers keep the event explicitly disclosed instead.
if (wasPersisted && !job.owner)
return false;
const durable = persistJobRecord(job, job.owner ?? PERSIST_OWNER);
// In non-panel mode the in-memory object is the authoritative status store. In panel
// mode a failed persist leaves the old downloading snapshot visible, so the caller must
// keep the event explicitly reconciled instead of emitting an unflagged FAILED row.
return durable || !persistedRecordsEnabled();
}
/** Rebuild an in-memory DownloadJob view from a persisted record (#529 adoption
* after a reconnect). It is a read-only snapshot — there is no live AbortController
* in THIS process for a job another/previous session started, so it can be polled
* by download_model action:"status" but not cancelled from here. */
function jobFromPersisted(rec) {
return {
id: rec.id,
trayId: rec.trayId,
progressId: rec.progressId,
partialPath: rec.partialPath,
partialIdentity: rec.partial_identity,
target: rec.target,
owner: rec.owner,
url: rec.url,
target_subfolder: rec.target_subfolder,
filename: rec.filename,
status: rec.status,
updated: rec.updated,
path: rec.path,
error: rec.error,
started_at: rec.started_at,
finished_at: rec.finished_at,
notes: rec.notes,
destKey: rec.dest_key,
reqKey: rec.req_key,
viaManager: rec.via_manager,
downloadRoute: rec.download_route,
resume: rec.resume,
live_visible: rec.live_visible,
verify_note: rec.verify_note,
disk_verified: rec.disk_verified,
verified_root: rec.verified_root,
staleInflight: rec.staleInflight,
staleForMs: rec.staleForMs,
// #1479 — the cancel path already probed this and status never asked, so the two
// actions answered the same question with opposite verdicts. Carried as `true` or
// absent: `writerProcessGone` returns undefined when it cannot tell, and that must
// keep the cautious note rather than becoming a death claim.
writerProvenGone: writerProcessGone(rec) === true ? true : undefined,
reclaimedDead: rec.reclaimed_dead,
interruptedByRestart: rec.interrupted_by_restart,
};
}
/** True when a FRESH foreign in-flight persisted record shares `id` but carries a
* DIFFERENT trayId — a distinct concurrent physical download in ANOTHER session (two
* distinct URLs resolving to the same dest+auth). The id alone can't disambiguate, so
* a by-id lookup/cancel must decline rather than silently act on the local one.
* Excludes this process's own records (same owner) and heartbeat-stale ones. */
function hasAmbiguousForeignSibling(id, localTrayId, localTarget) {
const now = Date.now();
return listPersistedDownloadJobs().some((rec) => rec.id === id &&
(rec.trayId !== localTrayId || rec.target !== localTarget) &&
rec.owner !== PERSIST_OWNER &&
rec.status === "downloading" &&
now - (rec.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS);
}
/** " (verified on disk)" only when the post-landing stat actually succeeded. A job
* whose file vanished between the rename and the check keeps its path but must not
* carry a disk-verification claim (codex gate, round 9). */
/** Do two absolute root paths name the same directory? Windows paths are
* case-insensitive and mix separators, so `C:\ComfyUI\models` and
* `c:/comfyui/models` are the SAME install — comparing them raw produced a false
* "a DIFFERENT install" downgrade of a correct verdict (codex gate, round 17). */
function sameRoot(a, b) {
if (!a || !b)
return false;
const norm = (s) => {
const slashed = s.replace(/\\/g, "/").replace(/\/+$/, "");
return process.platform === "win32" ? slashed.toLowerCase() : slashed;
};
return norm(a) === norm(b);
}
function diskQualifier(job) {
return job.disk_verified === false ? " (NOT found on disk when checked)" : " (verified on disk)";
}
export function describePlacement(job,
/** The models root the CONNECTED server reads right now (currentLiveModelsRoot()).
* A stored `visible` verdict may ONLY be re-asserted when this is present and
* names the same root the verdict was made against — see below. */
ctx) {
if (!job.viaManager &&
job.live_visible === "visible" &&
!sameRoot(ctx?.liveModelsDir, job.verified_root)) {
// A CONFIRMED rendering requires the READER to have re-established the verdict,
// so every way of failing to do that lands here (codex gate, rounds 11, 16, 18):
// - the verdict names a DIFFERENT root than the server now reads;
// - the verdict carries NO root (made against a destination only local
// configuration could vouch for);
// - and — the inversion this closes — there is NO current observation at all,
// because the probe transiently failed or resolved non-authoritatively.
// That last case previously fell through to "confirmed", reading a MISSING
// observation as "no contradiction". It is not: nothing verified that the server
// answering NOW can read this file. Absence of evidence renders as unverified.
return {
confirmed: false,
wrongPlace: false,
pathLabel: "written to",
pathQualifier: diskQualifier(job),
warning: !ctx?.liveModelsDir
? "this download was confirmed earlier, but the connected ComfyUI could not be asked " +
"just now which models directory it reads, so that confirmation cannot be re-established " +
"for the server you are connected to. Re-check with download_model action:\"status\", or confirm with " +
"list_local_models."
: job.verified_root
? `this download was verified against a ComfyUI reading "${job.verified_root}", but the ` +
`connected server now reads "${ctx.liveModelsDir}" — a DIFFERENT install. The earlier ` +
"confirmation does not apply to it; check list_local_models against the server you are " +
"connected to now."
: "this download's placement was confirmed against a models directory that could only be " +
`inferred from local configuration, and the connected server now reports "${ctx.liveModelsDir}" ` +
"as the directory it reads. The earlier confirmation cannot be re-asserted for it; " +
"check list_local_models against the server you are connected to now.",
};
}
if (job.viaManager) {
// #1374 — REPORT WHAT WAS OBSERVED, when anything was.
//
// This arm used to return one static sentence for every Manager outcome. It
// was true — the queue reports 'done' on a drain, not on a landing — and
// precisely because it was true of every outcome it distinguished none of
// them. A reporter whose Manager REFUSED the fetch on security_level read
// `done` and went looking for a 13.2 GB file that was never fetched.
//
// `verifyManagerVisibility` now runs on this route (see startDownloadJob), so
// there is usually a real answer to print. `confirmed` stays FALSE in every
// arm: a listing proves a file of that NAME exists somewhere the server
// reads, which is placement and not validity (#473 — a login page saved as a
// .safetensors lists perfectly happily).
if (job.live_visible === "not-visible") {
return {
confirmed: false,
// #1374 review, P1-4 — NOT `wrongPlace`, AND THE DIFFERENCE IS NOT
// COSMETIC.
//
// `wrongPlace` means DEMONSTRABLY misplaced, and its consumers act on it:
// apply_manifest renders it as `failed`. On the LOCAL route that is
// earned — the file was stat'd on disk, so "present but unlisted" is a
// real contradiction. Here nothing was stat'd at all. A Manager dispatch
// returns on ACCEPTANCE (and #1197: its queue can drain hours before the
// transfer does), so "not listed yet" is equally a 13 GB fetch still in
// flight or a stale listing. Calling that `failed` invents a failure, and
// a caller who believes it re-issues the download and pays for the
// transfer twice — the standing rule here is that a false failure costs
// more than an unconfirmed success.
//
// The finding is not softened, only correctly typed: the warning below
// states it in full, and `confirmed` stays false, so the item settles as
// PENDING — "the placement has not been established" — which is exactly
// what was observed.
wrongPlace: false,
pathLabel: "requested destination",
pathQualifier: "",
warning: `the dispatch was ACCEPTED and the connected ComfyUI does NOT list this file — ` +
`${job.verify_note ?? "the running server does not list this entry."} ` +
`ComfyUI-Manager reports its queue task 'done' even when it refused the fetch ` +
`outright (a security_level / network_mode rejection is the common cause), so a ` +
`'done' here is not evidence the file exists.`,
};
}
if (job.live_visible === "visible") {
return {
confirmed: false,
wrongPlace: false,
pathLabel: "requested destination",
pathQualifier: " (the connected ComfyUI lists it)",
warning: `the connected ComfyUI lists this file, so the dispatch landed somewhere it reads — ` +
`but that is PLACEMENT, not validity. ${job.verify_note ?? ""}`.trim(),
};
}
return {
confirmed: false,
wrongPlace: false,
pathLabel: "requested destination",
pathQualifier: "",
warning: "the dispatch was ACCEPTED, NOT verified as landed — ComfyUI-Manager reports its queue " +
"task 'done' even on failure, so this does not guarantee the file is present. " +
(job.verify_note ? `${job.verify_note} ` : "") +
"Confirm with list_local_models before relying on it.",
};
}
switch (job.live_visible) {
case "visible":
return {
confirmed: true,
wrongPlace: false,
pathLabel: "landed at",
pathQualifier: " (verified on disk, and the connected ComfyUI lists it)",
};
case "not-visible":
return {
confirmed: false,
wrongPlace: true,
pathLabel: "written to",
pathQualifier: diskQualifier(job),
warning: `NOT VISIBLE to the connected ComfyUI — ${job.verify_note ?? "the running server does not list this file."}`,
};
case "unknown":
return {
confirmed: false,
wrongPlace: false,
pathLabel: "written to",
pathQualifier: diskQualifier(job),
warning: `visibility to the connected ComfyUI is UNCONFIRMED${job.verify_note ? ` — ${job.verify_note}` : ""}. Check list_local_models before relying on it.`,
};
default:
// "pending" and undefined (pre-fix record) alike: the file landed, the check
// has not concluded. Never rendered as a confirmed success.
return {
confirmed: false,
wrongPlace: false,
pathLabel: "written to",
pathQualifier: "",
warning: "the file was materialized locally, but placement has NOT been confirmed yet — the " +
"check against the connected ComfyUI has not completed. Re-check with download_model action:\"status\", " +
"or confirm with list_local_models.",
};
}
}
/**
* Every DISTINCT tracked download that answers to `id`, across this process's
* registry AND the persisted store, deduped by the TRUE composite identity
* (id, trayId, target) — the same target-aware key `listDownloadJobs` uses.
*
* `id` is derived from destination+auth, so two different SOURCE URLs landing on
* one file deliberately share it (#467 P1-A dedup). That makes `id` a
* DESTINATION handle, not a job handle — and #822 is the consequence: an
* identifier that can name two rows cannot select either. This function is what
* lets every id-keyed operation SAY SO — listing the candidates a caller must
* choose between — instead of silently acting on one or reporting "not found".
*
* Newest-started first, so a caller printing candidates leads with the live one.
*/
export function listDownloadJobCandidates(id) {
const keyOf = (j) => `${j.id}\n${j.trayId}\n${j.target ?? ""}`;
const byKey = new Map();
for (const e of new Set(jobs.values())) {
if (e.job.id !== id)
continue;
const cur = byKey.get(keyOf(e.job));
if (!cur || (e.job.status === "done" && cur.status !== "done"))
byKey.set(keyOf(e.job), e.job);
}
// `trayId` is a hash of the URL, so it does NOT separate sessions: this
// session's SETTLED record for a URL and ANOTHER session's still-running
// download of the same URL to the same destination collapse to one target-aware
// key. Which
// one is reported matters — showing the local "cancelled" row while a foreign
// transfer is live would hide a running download behind a stale verdict, and
// that is worse than the ambiguity this function exists to surface.
const now = Date.now();
const isLive = (r) => r.status === "downloading" && now - (r.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS;
for (const rec of listPersistedDownloadJobs()) {
if (rec.id !== id)
continue;
const job = jobFromPersisted(rec);
const k = keyOf(job);
const cur = byKey.get(k);
if (!cur) {
byKey.set(k, job);
continue;
}
// Precedence, most authoritative first:
// 1. a validated DONE — the file landed, and no later verdict undoes that;
// 2. a LIVE transfer — still happening, so it is what the caller needs to know;
// 3. anything settled.
// (A live IN-MEMORY job is this process's own and already sits in `byKey`, so
// it is never displaced by a merely-settled persisted record.)
if (job.status === "done" && cur.status !== "done" && cur.status !== "downloading") {
byKey.set(k, job);
}
else if (isLive(rec) && cur.status !== "done" && cur.status !== "downloading") {
byKey.set(k, job);
}
}
// #1208 — a DETERMINISTIC tiebreak. started_at is a millisecond timestamp, so
// two jobs begun in the same millisecond compared equal and the order fell back
// to Map insertion order: stable locally, evidently not on a loaded CI runner.
// "lists newest first" failed on all THREE platforms at once and went green on
// an unchanged re-run — and a flake that fails everywhere simultaneously reads
// like a regression, which is what it costs to rule out.
//
// trayId is the right tiebreak: it is unique per physical download (two URLs
// sharing an id differ there), so equal timestamps order identically on every
// run and in every process.
return [...byKey.values()].sort((a, b) => b.started_at - a.started_at || compareTrayIds(a.trayId, b.trayId));
}
/**
* Resolve ONE download.
*
* `trayId` is the DISAMBIGUATOR (#822): with it the lookup keys on the full
* composite identity (id, trayId, target) and can name exactly one row when the
* current target is known. Without
* it, an id that denotes more than one download still resolves to nothing —
* but callers can now distinguish that from "no such download" by asking
* {@link listDownloadJobCandidates}, which is the difference between an
* actionable "say which one" and a false "it never existed".
*/
export function getDownloadJob(id, trayId, target) {
const targetSelector = target ?? currentDownloadTarget();
const candidates = listDownloadJobCandidates(id).filter((j) => (!trayId || j.trayId === trayId) &&
(targetSelector === undefined || j.target === targetSelector));
const selectOne = (rows) => {
const identities = new Set(rows.map((j) => `${j.id}\n${j.trayId}\n${j.target ?? ""}`));
return identities.size === 1 ? rows[0] : undefined;
};
if (trayId)
return selectOne(candidates);
// The registry indexes each job under its request key and (when local) its
// destination key; the public id is one of those, so a direct get resolves it.
const live = jobs.get(id)?.job;
// A LIVE in-flight job in THIS process is authoritative (it's actively writing here) —
// but decline if a concurrent live FOREIGN session shares the id with a different
// (trayId, target) identity.
if (live &&
(targetSelector === undefined || live.target === targetSelector) &&
live.status === "downloading") {
if (hasAmbiguousForeignSibling(id, live.trayId, live.target))
return undefined;
return live;
}
// Otherwise resolve across this session's TERMINAL in-memory record AND the persisted
// store (#529), honoring the INTEGRITY TRUTH: a validated DONE (file landed) — from
// ANY session, in-memory or persisted — must WIN over a cancelled/error record for the
// same id. A cancel/error never lands a file, so it must never mask another session's
// validated-complete file (cancelled-over-complete), and this holds whether the
// cancelled/error record is in-memory or persisted.
if (live &&
(targetSelector === undefined || live.target === targetSelector) &&
live.status === "done")
return live;
const now = Date.now();
const matches = listPersistedDownloadJobs().filter((r) => r.id === id && (targetSelector === undefined || r.target === targetSelector));
const freshIdentities = new Set(matches
.filter((r) => r.status === "downloading" && now - (r.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS)
.map((r) => `${r.id}\n${r.trayId}\n${r.target ?? ""}`));
const settledIdentities = new Set(matches
.filter((r) => r.status !== "downloading")
.map((r) => `${r.id}\n${r.trayId}\n${r.target ?? ""}`));
if (freshIdentities.size > 1 || (freshIdentities.size === 0 && settledIdentities.size > 1)) {
return undefined;
}
const persistedDone = matches
.filter((r) => r.status === "done")
.sort((a, b) => (b.updated ?? 0) - (a.updated ?? 0))[0];
if (persistedDone)
return jobFromPersisted(persistedDone);
// No DONE anywhere. Prefer a fresh live foreign in-flight download (it's still running
// elsewhere) — declining if ambiguous by trayId — then this session's own terminal
// record, then the newest persisted terminal.
const foreignLive = matches.filter((r) => r.status === "downloading" && now - (r.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS);
if (foreignLive.length > 0) {
if (new Set(foreignLive.map((r) => `${r.trayId}\n${r.target ?? ""}`)).size > 1)
return undefined; // ambiguous live
return jobFromPersisted(foreignLive.sort((a, b) => (b.updated ?? 0) - (a.updated ?? 0))[0]);
}
if (live &&
(targetSelector === undefined || live.target === targetSelector))
return live; // this session's terminal (cancelled/error), no DONE to prefer
const terminal = matches.sort((a, b) => (b.updated ?? 0) - (a.updated ?? 0))[0];
return terminal ? jobFromPersisted(terminal) : undefined;
}
/**
* Why a by-id lookup came back empty — when the answer is NOT "it is gone" (#1183).
*
* A reporter polled a live 26.5GB local download for ~30 minutes (20% → 95%) and
* then got, for ONE poll:
*
* No download matching id `cb43b8c206bec9b1`. It has either finished long ago
* … or never started
*
* The file existed nowhere on disk and nothing had been cancelled. Re-issuing the
* same URL immediately re-adopted the SAME id at 97% — the job was alive in
* process the entire time.
*
* getDownloadJob DECLINES in several places rather than guessing, and those
* declines are correct: two concurrent physical transfers sharing an id must not
* be resolved by picking one. What is not correct is rendering a decline as an
* ABSENCE. "I will not choose between two" and "there is nothing here" are
* different facts, and only the second justifies telling a user their download
* vanished — which is what invites a redundant multi-gigabyte re-download.
*
* So: when the lookup misses, ask the cheaper question it never asked — is there
* a LIVE record for this id at all? Returns undefined when there genuinely is
* not, so a real "not found" keeps its existing wording.
*/
export function describeUnresolvedDownload(id) {
const now = Date.now();
const targetSelector = currentDownloadTarget();
const liveInMemoryCandidate = jobs.get(id)?.job;
const liveInMemory = liveInMemoryCandidate &&
(targetSelector === undefined || liveInMemoryCandidate.target === targetSelector)
? liveInMemoryCandidate
: undefined;
const inFlight = listPersistedDownloadJobs().filter((r) => r.id === id &&
r.status === "downloading" &&
(targetSelector === undefined || r.target === targetSelector));
const livePersisted = inFlight.filter((r) => now - (r.updated ?? 0) < PERSISTED_INFLIGHT_STALE_MS);
// A STALE in-flight record counts here, hedged — and getting that wrong is
// what would have left the REPORTED case uncovered.
//
// This function first used the same freshness window the lookup uses, which
// means it went silent in exactly the situation most likely to have produced a
// one-poll miss during a 26GB transfer: a heartbeat delayed past 60s while the
// writer was busy. The codebase already states the right rule one file over
// (#761): "a missed heartbeat is only a liveness hint, not proof the transfer
// stopped" — which is why an in-flight record is retained for 6h rather than
// reaped at 60s. Declining to mention such a record would repeat the very fold
// this fix exists to remove, one level down.
const stalePersisted = inFlight.filter((r) => now - (r.updated ?? 0) >= PERSISTED_INFLIGHT_STALE_MS);
const inMemoryLive = liveInMemory?.status === "downloading";
if (!inMemoryLive && livePersisted.length === 0 && stalePersisted.length === 0) {
return undefined;
}
// Only a stale record to go on: say what is known and what is not, rather than
// asserting either "running" or "gone".
if (!inMemoryLive && livePersisted.length === 0) {
const newest = stalePersisted.sort((a, b) => (b.updated ?? 0) - (a.updated ?? 0))[0];
const ageS = Math.round((now - (newest?.updated ?? now)) / 1000);
return (`A download