@langchain/langgraph-sdk/dist/client/stream/media.js

Client library for interacting with the LangGraph API

//#region src/client/stream/media.ts
const MEDIA_BLOCK_TYPES = new Set([
	"audio",
	"image",
	"video",
	"file"
]);
/**
* Typed error thrown through `media.stream` / rejected from
* `media.blob` / `media.objectURL` when a handle fails before its
* message completes. Carries the bytes accumulated up to the failure
* point on `partialBytes` for callers that want to salvage or diagnose.
*/
var MediaAssemblyError = class extends Error {
	kind;
	messageId;
	partialBytes;
	cause;
	constructor(kind, messageId, partialBytes, message, options) {
		super(message ?? `media ${kind} for message ${messageId}`);
		this.name = "MediaAssemblyError";
		this.kind = kind;
		this.messageId = messageId;
		this.partialBytes = partialBytes;
		this.cause = options?.cause;
	}
};
function base64ToBytes(b64) {
	const binary = atob(b64);
	const bytes = new Uint8Array(binary.length);
	for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
	return bytes;
}
function concatBytes(parts, totalLength) {
	const out = new Uint8Array(totalLength);
	let offset = 0;
	for (const part of parts) {
		out.set(part, offset);
		offset += part.byteLength;
	}
	return out;
}
/**
* Concrete handle implementation shared by all four media types.
*
* One instance per `(messageId, blockType)` pair created by the
* assembler on first matching `content-block-start`.
*/
var MediaHandleImpl = class {
	type;
	messageId;
	namespace;
	node;
	id;
	mimeType;
	url;
	width;
	height;
	filename;
	monotonic = true;
	error;
	#parts = [];
	#totalBytes = 0;
	#partialSnapshot = new Uint8Array(0);
	#stream;
	#streamController;
	#blobResolve;
	#blobReject;
	#blobPromise;
	#transcriptParts = [];
	#transcriptResolve;
	#transcriptReject;
	#transcriptPromise;
	#cachedObjectURL;
	#urlSourced = false;
	#urlFetchPromise;
	#lastIndex = -1;
	#finished = false;
	#settled = false;
	#fetchImpl;
	constructor(options) {
		this.type = options.type;
		this.messageId = options.messageId;
		this.namespace = options.namespace;
		this.node = options.node;
		this.id = options.id;
		this.mimeType = options.mimeType;
		this.url = options.url;
		this.#fetchImpl = options.fetch;
		this.#blobPromise = new Promise((resolve, reject) => {
			this.#blobResolve = resolve;
			this.#blobReject = reject;
		});
		this.#blobPromise.catch(() => void 0);
		this.#transcriptPromise = new Promise((resolve, reject) => {
			this.#transcriptResolve = resolve;
			this.#transcriptReject = reject;
		});
		this.#transcriptPromise.catch(() => void 0);
	}
	/** Track a block index for the monotonic-ordering diagnostic. */
	observeIndex(index) {
		if (index !== this.#lastIndex + 1 && index !== this.#lastIndex) this.monotonic = false;
		if (index > this.#lastIndex) this.#lastIndex = index;
	}
	/** Absorb `mime_type` / per-type extras carried on an incoming block. */
	absorbBlock(block) {
		if (this.#urlSourced) return;
		if (block.type === "audio") this.#absorbAudio(block);
		else if (block.type === "image") this.#absorbImage(block);
		else if (block.type === "video") this.#absorbVideo(block);
		else if (block.type === "file") this.#absorbFile(block);
	}
	/** Record that the originating block arrived with `url` not `data`. */
	enterUrlMode(url) {
		this.#urlSourced = true;
		this.url = url;
	}
	/** Push a fresh chunk of bytes into the handle. */
	pushBytes(bytes) {
		if (this.#finished || this.#settled) return;
		if (bytes.byteLength === 0) return;
		this.#parts.push(bytes);
		this.#totalBytes += bytes.byteLength;
		this.#partialSnapshot = concatBytes(this.#parts, this.#totalBytes);
		if (this.#streamController != null) try {
			this.#streamController.enqueue(bytes);
		} catch {}
	}
	/** Append a transcript fragment from an audio block. */
	pushTranscript(fragment) {
		if (this.type !== "audio") return;
		if (this.#finished || this.#settled) return;
		if (fragment.length === 0) return;
		this.#transcriptParts.push(fragment);
	}
	/** Called on `message-finish`. Settles blob/transcript/stream. */
	finish() {
		if (this.#finished || this.#settled) return;
		this.#finished = true;
		this.#settled = true;
		const blob = new Blob([this.#partialSnapshot], { type: this.mimeType ?? "" });
		this.#blobResolve(blob);
		this.#transcriptResolve(this.#transcriptParts.length === 0 ? void 0 : this.#transcriptParts.join(""));
		try {
			this.#streamController?.close();
		} catch {}
	}
	/** Propagate an error through blob/transcript/stream. */
	fail(kind, reason, cause) {
		if (this.#settled) return this.error ?? new MediaAssemblyError(kind, this.messageId, this.#partialSnapshot, reason, { cause });
		this.#settled = true;
		const err = new MediaAssemblyError(kind, this.messageId, this.#partialSnapshot, reason, { cause });
		this.error = err;
		this.#blobReject(err);
		this.#transcriptReject(err);
		try {
			this.#streamController?.error(err);
		} catch {}
		return err;
	}
	get partialBytes() {
		return this.#partialSnapshot;
	}
	get blob() {
		if (this.#urlSourced) return this.#fetchUrlSourced().then((bytes) => new Blob([bytes], { type: this.mimeType ?? "" }));
		return this.#blobPromise;
	}
	get transcript() {
		return this.#transcriptPromise;
	}
	get objectURL() {
		if (this.#cachedObjectURL != null) {
			const cached = this.#cachedObjectURL;
			return Promise.resolve(cached);
		}
		return this.blob.then((blob) => {
			if (this.#cachedObjectURL != null) return this.#cachedObjectURL;
			const url = URL.createObjectURL(blob);
			this.#cachedObjectURL = url;
			return url;
		});
	}
	revoke() {
		const url = this.#cachedObjectURL;
		if (url == null) return;
		this.#cachedObjectURL = void 0;
		try {
			URL.revokeObjectURL(url);
		} catch {}
	}
	get stream() {
		if (this.#stream != null) return this.#stream;
		if (this.#urlSourced) return this.#buildUrlStream();
		return this.#buildInlineStream();
	}
	#absorbAudio(block) {
		const mimeType = block.mime_type ?? block.mimeType;
		if (this.mimeType == null && mimeType != null) this.mimeType = mimeType;
		if (block.transcript != null && block.transcript.length > 0) this.pushTranscript(block.transcript);
	}
	#absorbImage(block) {
		const mimeType = block.mime_type ?? block.mimeType;
		if (this.mimeType == null && mimeType != null) this.mimeType = mimeType;
		if (this.width == null && block.width != null) this.width = block.width;
		if (this.height == null && block.height != null) this.height = block.height;
	}
	#absorbVideo(block) {
		const mimeType = block.mime_type ?? block.mimeType;
		if (this.mimeType == null && mimeType != null) this.mimeType = mimeType;
	}
	#absorbFile(block) {
		const mimeType = block.mime_type ?? block.mimeType;
		if (this.mimeType == null && mimeType != null) this.mimeType = mimeType;
		if (this.filename == null && block.filename != null) this.filename = block.filename;
	}
	#buildInlineStream() {
		const seed = this.#partialSnapshot;
		const alreadyFinished = this.#finished;
		const alreadyErrored = this.error;
		this.#stream = new ReadableStream({
			start: (controller) => {
				this.#streamController = controller;
				if (seed.byteLength > 0) controller.enqueue(seed);
				if (alreadyErrored != null) {
					controller.error(alreadyErrored);
					return;
				}
				if (alreadyFinished) controller.close();
			},
			cancel: () => {
				this.#streamController = void 0;
			}
		});
		return this.#stream;
	}
	#buildUrlStream() {
		const urlSourceFetch = this.#startUrlFetch();
		this.#stream = new ReadableStream({
			start: async (controller) => {
				try {
					const response = await urlSourceFetch;
					if (response.body == null) {
						const bytes = new Uint8Array(await response.arrayBuffer());
						if (bytes.byteLength > 0) controller.enqueue(bytes);
						controller.close();
						return;
					}
					const reader = response.body.getReader();
					while (true) {
						const { done, value } = await reader.read();
						if (done) break;
						if (value != null) controller.enqueue(value);
					}
					controller.close();
				} catch (err) {
					controller.error(this.fail("fetch-failed", err?.message, err));
				}
			},
			cancel: () => {
				this.#streamController = void 0;
			}
		});
		return this.#stream;
	}
	/** Memoised fetch for URL-sourced blocks — returns one `Response`. */
	#startUrlFetch() {
		const url = this.url;
		return this.#fetchImpl(url).then((response) => {
			if (!response.ok) throw new Error(`fetch(${url}) failed: ${response.status} ${response.statusText}`);
			return response;
		});
	}
	/** Fetch + buffer for URL-sourced `blob` access. Memoised. */
	#fetchUrlSourced() {
		if (this.#urlFetchPromise != null) return this.#urlFetchPromise;
		this.#urlFetchPromise = (async () => {
			try {
				const response = await this.#startUrlFetch();
				const bytes = new Uint8Array(await response.arrayBuffer());
				this.#parts.length = 0;
				this.#parts.push(bytes);
				this.#totalBytes = bytes.byteLength;
				this.#partialSnapshot = bytes;
				this.#finished = true;
				this.#settled = true;
				return bytes;
			} catch (err) {
				throw this.fail("fetch-failed", err?.message, err);
			}
		})();
		return this.#urlFetchPromise;
	}
};
var MediaAssembler = class {
	#callbacks;
	#fetch;
	#active = /* @__PURE__ */ new Map();
	#activeByNamespaceNode = /* @__PURE__ */ new Map();
	#syntheticCounter = 0;
	constructor(options = {}) {
		this.#callbacks = options;
		if (options.fetch != null) this.#fetch = options.fetch;
		else if (typeof fetch === "function") this.#fetch = fetch;
		else this.#fetch = () => {
			throw new Error("MediaAssembler: no fetch implementation available. Pass `fetch` in options.");
		};
	}
	/**
	* Fold a single `messages` event. Non-media blocks and
	* informational events (e.g. `content-block-finish`) are no-ops.
	*/
	consume(event) {
		const data = event.params.data;
		const namespace = event.params.namespace;
		const node = event.params.node;
		const nsNodeKey = `${namespace.join("/")}::${node ?? ""}`;
		if (data.event === "message-start") {
			this.#flushSlot(nsNodeKey, "finish");
			this.#activeByNamespaceNode.set(nsNodeKey, {
				messageId: data.id ?? "",
				keys: /* @__PURE__ */ new Set(),
				indexKeys: /* @__PURE__ */ new Map()
			});
			return;
		}
		if (data.event === "message-finish") {
			this.#flushSlot(nsNodeKey, "finish");
			this.#activeByNamespaceNode.delete(nsNodeKey);
			return;
		}
		if (data.event === "error") {
			this.#flushSlot(nsNodeKey, "error", data.message);
			this.#activeByNamespaceNode.delete(nsNodeKey);
			return;
		}
		if (data.event !== "content-block-start" && data.event !== "content-block-delta" && data.event !== "content-block-finish") return;
		const block = data.content;
		const blockIndex = data.index ?? 0;
		let active = this.#activeByNamespaceNode.get(nsNodeKey);
		if (active == null) {
			active = {
				messageId: `__synthetic_${++this.#syntheticCounter}`,
				keys: /* @__PURE__ */ new Set(),
				indexKeys: /* @__PURE__ */ new Map()
			};
			this.#activeByNamespaceNode.set(nsNodeKey, active);
		}
		if (block == null && data.event === "content-block-delta") {
			const delta = data.delta;
			const deltaKey = active.indexKeys.get(blockIndex);
			const deltaHandle = deltaKey != null ? this.#active.get(deltaKey) : void 0;
			if (delta == null || typeof delta !== "object") return;
			const record = delta;
			if (deltaHandle == null) {
				if (record.type !== "block-delta" || record.fields == null || typeof record.fields !== "object") return;
				const fields = record.fields;
				if (!MEDIA_BLOCK_TYPES.has(fields.type)) return;
				this.#consumeMediaBlock({
					active,
					block: fields,
					blockIndex,
					dataEvent: data.event,
					namespace,
					node,
					terminal: false,
					createIfMissing: true
				});
				return;
			}
			deltaHandle.observeIndex(blockIndex);
			if (record.type === "data-delta" && typeof record.data === "string") {
				try {
					deltaHandle.pushBytes(base64ToBytes(record.data));
				} catch (err) {
					deltaHandle.fail("message-error", "invalid base64 on delta", err);
				}
				return;
			}
			if (record.type === "block-delta" && record.fields != null && typeof record.fields === "object") {
				const fields = record.fields;
				deltaHandle.absorbBlock(fields);
				if (!deltaHandle.error && fields.data != null) try {
					deltaHandle.pushBytes(base64ToBytes(fields.data));
				} catch (err) {
					deltaHandle.fail("message-error", "invalid base64 on delta", err);
				}
			}
			return;
		}
		if (block == null) return;
		const blockType = block.type;
		if (!MEDIA_BLOCK_TYPES.has(blockType)) return;
		this.#consumeMediaBlock({
			active,
			block,
			blockIndex,
			dataEvent: data.event,
			namespace,
			node,
			terminal: data.event === "content-block-finish",
			createIfMissing: data.event === "content-block-start" || data.event === "content-block-finish"
		});
	}
	#consumeMediaBlock({ active, block, blockIndex, dataEvent, namespace, node, terminal, createIfMissing }) {
		const blockType = block.type;
		if (!MEDIA_BLOCK_TYPES.has(blockType)) return;
		const mediaType = blockType;
		const key = `${active.messageId}::${mediaType}::${blockIndex}`;
		let handle = this.#active.get(key);
		const isStart = dataEvent === "content-block-start";
		if (handle == null) {
			const isTerminalBlock = terminal;
			if (!isStart && !isTerminalBlock && !createIfMissing) return;
			const mediaBlock = block;
			handle = new MediaHandleImpl({
				type: mediaType,
				messageId: active.messageId,
				namespace: [...namespace],
				node,
				id: mediaBlock.id,
				mimeType: mediaBlock.mime_type ?? mediaBlock.mimeType,
				url: mediaBlock.url != null && mediaBlock.data == null ? mediaBlock.url : void 0,
				fetch: this.#fetch
			});
			if (mediaBlock.url != null && mediaBlock.data == null) handle.enterUrlMode(mediaBlock.url);
			handle.observeIndex(blockIndex);
			handle.absorbBlock(block);
			if (mediaBlock.data != null) try {
				handle.pushBytes(base64ToBytes(mediaBlock.data));
			} catch (err) {
				handle.fail("message-error", "invalid base64 on initial block", err);
			}
			this.#active.set(key, handle);
			active.keys.add(key);
			active.indexKeys.set(blockIndex, key);
			this.#emit(handle);
			if (isTerminalBlock) {
				handle.finish();
				this.#active.delete(key);
				active.keys.delete(key);
				active.indexKeys.delete(blockIndex);
			}
			return;
		}
		if (terminal) return;
		const mediaBlock = block;
		handle.observeIndex(blockIndex);
		handle.absorbBlock(block);
		if (!handle.error && mediaBlock.data != null) try {
			handle.pushBytes(base64ToBytes(mediaBlock.data));
		} catch (err) {
			handle.fail("message-error", "invalid base64 on delta", err);
		}
	}
	/**
	* Finish or fail every media handle currently active under the
	* given `(namespace, node)` slot and clear its bookkeeping. Called
	* on `message-finish`, `error`, and when a new `message-start`
	* rebinds a still-open slot.
	*/
	#flushSlot(nsNodeKey, mode, errorMessage) {
		const active = this.#activeByNamespaceNode.get(nsNodeKey);
		if (active == null) return;
		for (const key of active.keys) {
			const handle = this.#active.get(key);
			if (handle != null) if (mode === "finish") handle.finish();
			else handle.fail("message-error", errorMessage);
			this.#active.delete(key);
		}
		active.keys.clear();
		active.indexKeys.clear();
	}
	/**
	* Abort all outstanding handles with a `stream-closed` error.
	* Called when the upstream event source terminates before the
	* messages it was assembling had a chance to finish.
	*/
	close() {
		for (const handle of this.#active.values()) handle.fail("stream-closed", "upstream event stream closed");
		this.#active.clear();
		this.#activeByNamespaceNode.clear();
	}
	#emit(handle) {
		switch (handle.type) {
			case "audio":
				this.#callbacks.onAudio?.(handle);
				break;
			case "image":
				this.#callbacks.onImage?.(handle);
				break;
			case "video":
				this.#callbacks.onVideo?.(handle);
				break;
			case "file":
				this.#callbacks.onFile?.(handle);
				break;
		}
		this.#callbacks.onMedia?.(handle);
	}
};
//#endregion
export { MediaAssembler, MediaAssemblyError };

//# sourceMappingURL=media.js.map