openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
554 lines (553 loc) • 23.4 kB
JavaScript
import { p as clampPositiveTimerTimeoutMs } from "./number-coercion-CLj0HTDM.js";
import { c as isRecord } from "./record-coerce-DItp3I4t.js";
import { m as resolveAgentWorkspaceDir } from "./agent-scope-config-DcbEhP0R.js";
import { d as isSecretRef } from "./types.secrets-kC0nOetj.js";
import { t as isPromiseLike } from "./promise-like-D7-l5Fsp.js";
import { s as normalizeModelRef } from "./model-ref-shared-Dz7QU0Lx.js";
import "./agent-scope-DbtJyKUL.js";
import { c as resolveProviderModelMaterializationAuthMode } from "./openai-model-routes-Bte2KVl5.js";
import { a as prepareProviderRuntimeAuth } from "./provider-runtime.runtime.js";
import { r as resolveProviderRequestCapabilities } from "./provider-attribution-CDS-94hQ.js";
import { a as getModelProviderRequestRouteFacts, o as getModelProviderRequestTransport, t as applyPreparedRuntimeAuthToModel } from "./provider-request-config-DIOYidiO.js";
import { a as unwrapSecretSentinelsForProviderEgress, i as unwrapModelHeaderSentinelsForProviderEgress, t as protectPreparedProviderRuntimeAuth } from "./provider-secret-egress-C-JiHB7J.js";
import { ft as getModelRegistryRuntime } from "./sessions-BdNAJTEP.js";
import { a as bindModelLlmRuntime, t as complete } from "./stream-Ci9pOvq7.js";
import { o as requireApiKey } from "./model-auth-runtime-shared-C48YoQY0.js";
import { t as acquireAgentRunPreparedModelRuntime } from "./prepared-model-runtime-DgNj94yb.js";
import { t as resolveApiKeyForProviderCore } from "./model-auth-provider-Dd_3dXeY.js";
import { i as getApiKeyForModelCore, r as applySecretRefHeaderSentinels } from "./model-auth-C48_DZ-I.js";
import "./model-selection-di2kjKCB.js";
import { t as registerProviderStreamForModel } from "./provider-stream-BNpXmhsR.js";
import { n as normalizeMediaProviderId } from "./provider-id-DSbuCFIb.js";
import { r as minimaxUnderstandImage, t as isMinimaxVlmModel } from "./minimax-vlm-BXnJAulK.js";
import { i as hasImageReasoningOnlyResponse, t as coerceImageAssistantText } from "./image-tool.helpers-DgXx3oYT.js";
import { n as resolveModelAsync } from "./model-qCT5QXYI.js";
import { r as providerUsesCredentialScopedModelMetadata } from "./credential-scoped-model-dyTrjhuq.js";
//#region src/media-understanding/image-model-runtime.ts
const resolvedImageRuntimeContexts = /* @__PURE__ */ new WeakMap();
function getResolvedImageRuntimeContext(model) {
return resolvedImageRuntimeContexts.get(model);
}
function bindResolvedImageRuntime(params, apiKey, model) {
resolvedImageRuntimeContexts.set(model, {
cfg: params.cfg,
agentDir: params.agentDir,
...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}
});
return {
runtimeValue: apiKey,
model
};
}
function formatModelInputCapabilities(input) {
return input && input.length > 0 ? input.join(", ") : "none";
}
function requireImageCapableModel(params) {
if (!params.model) throw new Error(`Unknown model: ${params.resolvedProvider}/${params.resolvedModel}`);
if (params.model.input?.includes("image")) return params.model;
if (isMinimaxVlmModel(params.resolvedProvider, params.resolvedModel)) throw new Error(`Unknown model: ${params.resolvedProvider}/${params.resolvedModel}`);
throw new Error(`Model does not support images: ${params.requestedProvider}/${params.requestedModel} (resolved ${params.model.provider}/${params.model.id} input: ${formatModelInputCapabilities(params.model.input)})`);
}
async function prepareResolvedImageRuntime(params, resolvedModel, authStorage, modelRegistry) {
let model = resolvedModel;
const modelRuntime = getModelRegistryRuntime(modelRegistry);
const apiKeyInfo = await getApiKeyForModelCore({
model,
cfg: params.cfg,
agentDir: params.agentDir,
...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {},
profileId: params.profile,
preferredProfile: params.preferredProfile,
store: params.authStore,
secretSentinels: true
});
if (providerUsesCredentialScopedModelMetadata({
provider: model.provider,
modelId: model.id,
config: params.cfg,
agentDir: params.agentDir,
workspaceDir: params.workspaceDir
})) {
const authProfileMode = resolveProviderModelMaterializationAuthMode(apiKeyInfo.mode);
model = requireImageCapableModel({
model: (await resolveModelAsync(model.provider, model.id, params.agentDir, params.cfg, {
authStorage,
modelRegistry,
skipAgentDiscovery: true,
allowBundledStaticCatalogFallback: true,
preparedModelRuntime: params.preparedModelRuntime,
...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {},
...apiKeyInfo.profileId ? { authProfileId: apiKeyInfo.profileId } : authProfileMode ? { authProfileMode } : {}
})).model,
resolvedProvider: model.provider,
resolvedModel: model.id,
requestedProvider: params.provider,
requestedModel: params.model
});
}
if (!apiKeyInfo.apiKey?.trim() && apiKeyInfo.mode === "aws-sdk" && model.api === "bedrock-converse-stream") return bindResolvedImageRuntime(params, "", bindModelLlmRuntime(applySecretRefHeaderSentinels(model, params.cfg), modelRuntime.llmRuntime));
let apiKey = requireApiKey(apiKeyInfo, model.provider);
const preparedAuth = protectPreparedProviderRuntimeAuth({
provider: model.provider,
preparedAuth: await prepareProviderRuntimeAuth({
provider: model.provider,
config: params.cfg,
workspaceDir: params.workspaceDir,
env: process.env,
context: {
config: params.cfg,
workspaceDir: params.workspaceDir,
env: process.env,
provider: model.provider,
modelId: model.id,
model,
apiKey,
authMode: apiKeyInfo.mode,
profileId: apiKeyInfo.profileId
}
})
});
apiKey = preparedAuth?.apiKey?.trim() || apiKey;
model = applyPreparedRuntimeAuthToModel(model, preparedAuth);
authStorage.setRuntimeApiKey(model.provider, apiKey);
return bindResolvedImageRuntime(params, apiKey, bindModelLlmRuntime(applySecretRefHeaderSentinels(model, params.cfg), modelRuntime.llmRuntime));
}
async function resolveImageRuntime(params) {
const resolvedRef = normalizeModelRef(params.provider, params.model);
const workspaceDir = params.workspaceDir ?? (params.agentId ? resolveAgentWorkspaceDir(params.cfg ?? {}, params.agentId) : void 0);
const runtimeParams = workspaceDir ? {
...params,
workspaceDir
} : params;
const authProfileOptions = {
...params.profile ? { authProfileId: params.profile } : {},
...params.preferredProfile ? { preferredProfile: params.preferredProfile } : {}
};
const preparedRuntimeLease = params.preparedModelRuntime ? {
snapshot: params.preparedModelRuntime,
release: () => {}
} : await acquireAgentRunPreparedModelRuntime({
agentDir: params.agentDir,
...params.agentId ? { agentId: params.agentId } : {},
config: params.cfg ?? {},
...runtimeParams.workspaceDir ? { workspaceDir: runtimeParams.workspaceDir } : {},
loadRuntimePlugins: true,
runtimePluginSelections: [{
provider: resolvedRef.provider,
modelId: resolvedRef.model,
...params.agentId ? { agentId: params.agentId } : {}
}]
}, { catalogMode: "static" });
let leaseRetained = false;
const retainLease = (resolved) => {
leaseRetained = true;
return {
...resolved,
release: preparedRuntimeLease.release
};
};
try {
const preparedRuntime = preparedRuntimeLease.snapshot;
const preparedWorkspaceDir = preparedRuntime.workspaceDir ?? runtimeParams.workspaceDir;
const preparedParams = {
...runtimeParams,
agentDir: preparedRuntime.agentDir,
cfg: preparedRuntime.config,
preparedModelRuntime: preparedRuntime,
...preparedWorkspaceDir ? { workspaceDir: preparedWorkspaceDir } : {}
};
const resolveOptions = {
allowBundledStaticCatalogFallback: true,
...preparedRuntime.createStores(),
preparedModelRuntime: preparedRuntime,
skipAgentDiscovery: true,
...preparedParams.workspaceDir ? { workspaceDir: preparedParams.workspaceDir } : {},
...authProfileOptions
};
const resolved = await resolveModelAsync(resolvedRef.provider, resolvedRef.model, preparedParams.agentDir, preparedParams.cfg, resolveOptions);
return retainLease(await prepareResolvedImageRuntime(preparedParams, requireImageCapableModel({
model: resolved.model,
resolvedProvider: resolvedRef.provider,
resolvedModel: resolvedRef.model,
requestedProvider: params.provider,
requestedModel: params.model
}), resolved.authStorage, resolved.modelRegistry));
} finally {
if (!leaseRetained) preparedRuntimeLease.release();
}
}
//#endregion
//#region src/media-understanding/image.ts
function resolveImageToolMaxTokens(modelMaxTokens, requestedMaxTokens = 4096) {
if (typeof modelMaxTokens !== "number" || !Number.isFinite(modelMaxTokens) || modelMaxTokens <= 0) return requestedMaxTokens;
return Math.min(requestedMaxTokens, modelMaxTokens);
}
function isNativeResponsesReasoningPayload(model) {
if (model.api !== "openai-responses" && model.api !== "azure-openai-responses" && model.api !== "openai-chatgpt-responses") return false;
return resolveProviderRequestCapabilities({
provider: model.provider,
api: model.api,
baseUrl: model.baseUrl,
capability: "image",
transport: "media-understanding",
providerMetadataOwners: getModelProviderRequestRouteFacts(model)?.providerMetadataOwners
}).usesKnownNativeOpenAIRoute;
}
function removeReasoningInclude(value) {
if (!Array.isArray(value)) return value;
const next = value.filter((entry) => entry !== "reasoning.encrypted_content");
return next.length > 0 ? next : void 0;
}
function disableReasoningForImageRetryPayload(payload, model) {
if (!isRecord(payload)) return;
const next = { ...payload };
delete next.reasoning;
delete next.reasoning_effort;
const include = removeReasoningInclude(next.include);
if (include === void 0) delete next.include;
else next.include = include;
if (isNativeResponsesReasoningPayload(model)) next.reasoning = { effort: "none" };
return next;
}
function isImageModelNoTextError(err) {
return err instanceof Error && /^Image model returned no text\b/.test(err.message);
}
function composeImageDescriptionPayloadHandlers(first, second) {
if (!first) return second;
if (!second) return first;
return (payload, payloadModel) => {
const runSecond = (firstResult) => {
const secondResult = second(firstResult === void 0 ? payload : firstResult, payloadModel);
const coerceResult = (resolvedSecond) => resolvedSecond === void 0 ? firstResult : resolvedSecond;
return isPromiseLike(secondResult) ? Promise.resolve(secondResult).then(coerceResult) : coerceResult(secondResult);
};
const firstResult = first(payload, payloadModel);
if (isPromiseLike(firstResult)) return Promise.resolve(firstResult).then(runSecond);
return runSecond(firstResult);
};
}
function buildImageContext(prompt, images, opts) {
const imageContent = images.map((image) => ({
type: "image",
data: image.buffer.toString("base64"),
mimeType: image.mime ?? "image/jpeg"
}));
const content = opts?.promptInUserContent ? [{
type: "text",
text: prompt
}, ...imageContent] : imageContent;
return {
...opts?.promptInUserContent ? {} : { systemPrompt: prompt },
messages: [{
role: "user",
content,
timestamp: Date.now()
}]
};
}
function shouldPlaceImagePromptInUserContent(model) {
if (model.provider === "github-copilot") return true;
const capabilities = resolveProviderRequestCapabilities({
provider: model.provider,
api: model.api,
baseUrl: model.baseUrl,
capability: "image",
transport: "media-understanding",
providerMetadataOwners: getModelProviderRequestRouteFacts(model)?.providerMetadataOwners
});
return capabilities.endpointClass === "openrouter" || capabilities.endpointClass === "modelstudio-native" || model.provider.toLowerCase() === "openrouter" && capabilities.endpointClass === "default";
}
function buildImageRequestHeaders(model) {
if (model.provider !== "github-copilot") return;
return {
"x-initiator": "user",
"Copilot-Vision-Request": "true"
};
}
async function describeImagesWithMinimax(params) {
const responses = [];
const apiKey = unwrapSecretSentinelsForProviderEgress(params.runtimeValue, "MiniMax VLM request");
for (const [index, image] of params.images.entries()) {
params.signal?.throwIfAborted();
const prompt = params.images.length > 1 ? `${params.prompt}\n\nDescribe image ${index + 1} of ${params.images.length} independently.` : params.prompt;
const text = await minimaxUnderstandImage({
apiKey,
provider: params.provider,
prompt,
imageDataUrl: `data:${image.mime ?? "image/jpeg"};base64,${image.buffer.toString("base64")}`,
modelBaseUrl: params.modelBaseUrl,
timeoutMs: params.timeoutMs,
allowPrivateNetwork: params.allowPrivateNetwork,
request: params.request,
signal: params.signal
});
responses.push(params.images.length > 1 ? `Image ${index + 1}:\n${text.trim()}` : text.trim());
}
return {
text: responses.join("\n\n").trim(),
model: params.modelId
};
}
function isUnknownModelError(err) {
return err instanceof Error && /^Unknown model:/i.test(err.message);
}
function resolveConfiguredProviderBaseUrl(cfg, provider) {
const direct = cfg.models?.providers?.[provider];
if (typeof direct?.baseUrl === "string" && direct.baseUrl.trim()) return direct.baseUrl.trim();
const normalizedProvider = normalizeMediaProviderId(provider);
const normalized = cfg.models?.providers?.[normalizedProvider];
if (typeof normalized?.baseUrl === "string" && normalized.baseUrl.trim()) {
if (isMinimaxCnAlias(provider) && !isMinimaxCnBaseUrl(normalized.baseUrl)) return;
return normalized.baseUrl.trim();
}
}
function resolveConfiguredProviderAllowPrivateNetwork(cfg, provider) {
const direct = cfg.models?.providers?.[provider]?.request?.allowPrivateNetwork;
if (typeof direct === "boolean") return direct;
const normalizedProvider = normalizeMediaProviderId(provider);
const normalized = cfg.models?.providers?.[normalizedProvider]?.request?.allowPrivateNetwork;
if (typeof normalized === "boolean") return normalized;
}
function isMinimaxCnAlias(provider) {
const normalized = provider.trim().toLowerCase();
return normalized === "minimax-cn" || normalized === "minimax-portal-cn";
}
function isMinimaxCnBaseUrl(baseUrl) {
const trimmed = baseUrl.trim();
if (!trimmed) return false;
try {
return new URL(/^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`).hostname.toLowerCase() === "api.minimaxi.com";
} catch {
return false;
}
}
function hasConfiguredProviderApiKey(cfg, provider) {
const apiKey = cfg.models?.providers?.[provider]?.apiKey;
return typeof apiKey === "string" && apiKey.trim().length > 0 || isSecretRef(apiKey);
}
function resolveMinimaxVlmAuthProvider(cfg, provider) {
if (!isMinimaxCnAlias(provider) || hasConfiguredProviderApiKey(cfg, provider)) return provider;
return normalizeMediaProviderId(provider);
}
async function resolveMinimaxVlmFallbackRuntime(params) {
const authProvider = resolveMinimaxVlmAuthProvider(params.cfg, params.provider);
const auth = await resolveApiKeyForProviderCore({
provider: authProvider,
cfg: params.cfg,
secretSentinels: true,
profileId: params.profile,
preferredProfile: params.preferredProfile,
agentDir: params.agentDir,
...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}
});
return {
runtimeValue: requireApiKey(auth, authProvider),
modelBaseUrl: resolveConfiguredProviderBaseUrl(params.cfg, params.provider)
};
}
function resolveImageDescriptionTimeoutMs(timeoutMs) {
return clampPositiveTimerTimeoutMs(timeoutMs);
}
function buildImageDescriptionTimeoutError(params) {
if (params.phase === "setup") return /* @__PURE__ */ new Error(`image description setup timed out after ${params.timeoutMs}ms before provider request started`);
const setupDurationMs = typeof params.setupDurationMs === "number" && Number.isFinite(params.setupDurationMs) ? Math.max(0, Math.floor(params.setupDurationMs)) : 0;
return /* @__PURE__ */ new Error(setupDurationMs > 0 ? `image description request timed out after ${params.timeoutMs}ms (setup took ${setupDurationMs}ms before provider request started)` : `image description request timed out after ${params.timeoutMs}ms`);
}
async function withImageDescriptionTimeout(params) {
params.signal?.throwIfAborted();
if (params.timeoutMs === void 0 && !params.signal) return await params.task;
let timeout;
let removeAbortListener;
const races = [params.task];
if (params.timeoutMs !== void 0) races.push(new Promise((_, reject) => {
timeout = setTimeout(() => {
params.controller.abort();
reject(params.createTimeoutError(params.timeoutMs));
}, params.timeoutMs);
}));
if (params.signal) races.push(new Promise((_, reject) => {
const onAbort = () => {
try {
params.signal?.throwIfAborted();
} catch (error) {
reject(error instanceof Error ? error : new Error("image description aborted", { cause: error }));
}
};
params.signal?.addEventListener("abort", onAbort, { once: true });
removeAbortListener = () => params.signal?.removeEventListener("abort", onAbort);
if (params.signal?.aborted) onAbort();
}));
try {
return await Promise.race(races);
} finally {
removeAbortListener?.();
if (timeout) clearTimeout(timeout);
}
}
async function describeImagesWithModelInternal(params, options = {}) {
const prompt = params.prompt ?? "Describe the image.";
params.signal?.throwIfAborted();
const startedAtMs = Date.now();
const controller = new AbortController();
const requestSignal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
const configuredTimeoutMs = resolveImageDescriptionTimeoutMs(params.timeoutMs);
const allowPrivateNetwork = resolveConfiguredProviderAllowPrivateNetwork(params.cfg, params.provider);
let runtimeValue;
let model;
let releaseRuntime;
const resolutionTask = resolveImageRuntime(params);
try {
const resolved = await withImageDescriptionTimeout({
controller,
signal: params.signal,
timeoutMs: configuredTimeoutMs,
createTimeoutError: (timeoutMs) => buildImageDescriptionTimeoutError({
phase: "setup",
timeoutMs
}),
task: resolutionTask
});
runtimeValue = resolved.runtimeValue;
model = resolved.model;
releaseRuntime = resolved.release;
} catch (err) {
resolutionTask.then((late) => late.release(), () => void 0);
params.signal?.throwIfAborted();
if (!isMinimaxVlmModel(params.provider, params.model) || !isUnknownModelError(err)) throw err;
const fallback = await withImageDescriptionTimeout({
controller,
signal: params.signal,
timeoutMs: configuredTimeoutMs,
createTimeoutError: (timeoutMs) => buildImageDescriptionTimeoutError({
phase: "setup",
timeoutMs
}),
task: resolveMinimaxVlmFallbackRuntime(params)
});
return await describeImagesWithMinimax({
runtimeValue: fallback.runtimeValue,
provider: params.provider,
modelId: params.model,
modelBaseUrl: fallback.modelBaseUrl,
prompt,
timeoutMs: params.timeoutMs,
images: params.images,
allowPrivateNetwork,
signal: params.signal
});
}
const apiKey = runtimeValue;
try {
params.signal?.throwIfAborted();
const setupDurationMs = Date.now() - startedAtMs;
if (isMinimaxVlmModel(model.provider, model.id)) return await describeImagesWithMinimax({
runtimeValue,
provider: model.provider,
modelId: model.id,
modelBaseUrl: model.baseUrl,
prompt,
timeoutMs: params.timeoutMs,
images: params.images,
request: getModelProviderRequestTransport(model),
signal: params.signal
});
const resolvedRuntimeContext = getResolvedImageRuntimeContext(model);
const requestModel = unwrapModelHeaderSentinelsForProviderEgress(model, "image description provider request");
const providerStreamFn = registerProviderStreamForModel({
model: requestModel,
cfg: resolvedRuntimeContext?.cfg ?? params.cfg,
agentDir: resolvedRuntimeContext?.agentDir ?? params.agentDir,
...resolvedRuntimeContext?.workspaceDir ? { workspaceDir: resolvedRuntimeContext.workspaceDir } : params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}
});
const context = buildImageContext(prompt, params.images, { promptInUserContent: shouldPlaceImagePromptInUserContent(model) });
const maxTokens = resolveImageToolMaxTokens(model.maxTokens, params.maxTokens);
const completeImage = async (onPayload) => {
params.signal?.throwIfAborted();
const payloadHandler = composeImageDescriptionPayloadHandlers(onPayload, options.onPayload);
const timeoutMs = configuredTimeoutMs;
const headers = buildImageRequestHeaders(requestModel);
const streamOptions = {
apiKey,
maxTokens,
signal: requestSignal,
...timeoutMs !== void 0 ? { timeoutMs } : {},
...headers ? { headers } : {},
...payloadHandler ? { onPayload: payloadHandler } : {}
};
const task = providerStreamFn ? (async () => await (await providerStreamFn(requestModel, context, streamOptions)).result())() : complete(requestModel, context, streamOptions);
return await withImageDescriptionTimeout({
controller,
signal: params.signal,
timeoutMs,
createTimeoutError: (requestTimeoutMs) => buildImageDescriptionTimeoutError({
phase: "request",
timeoutMs: requestTimeoutMs,
setupDurationMs
}),
task
});
};
const message = await completeImage();
try {
return {
text: coerceImageAssistantText({
message,
provider: model.provider,
model: model.id
}),
model: model.id
};
} catch (err) {
if (!isImageModelNoTextError(err) || !hasImageReasoningOnlyResponse(message)) throw err;
}
params.signal?.throwIfAborted();
const retryMessage = await completeImage(disableReasoningForImageRetryPayload);
return {
text: coerceImageAssistantText({
message: retryMessage,
provider: model.provider,
model: model.id
}),
model: model.id
};
} finally {
releaseRuntime?.();
}
}
function toImagesDescriptionRequest(params) {
return {
images: [{
buffer: params.buffer,
fileName: params.fileName,
mime: params.mime
}],
model: params.model,
provider: params.provider,
prompt: params.prompt,
maxTokens: params.maxTokens,
timeoutMs: params.timeoutMs,
...params.signal ? { signal: params.signal } : {},
profile: params.profile,
preferredProfile: params.preferredProfile,
authStore: params.authStore,
...params.agentId ? { agentId: params.agentId } : {},
agentDir: params.agentDir,
...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {},
...params.preparedModelRuntime ? { preparedModelRuntime: params.preparedModelRuntime } : {},
cfg: params.cfg
};
}
async function describeImagesWithModelCore(params) {
return await describeImagesWithModelInternal(params);
}
async function describeImagesWithModelPayloadTransformCore(params, onPayload) {
return await describeImagesWithModelInternal(params, { onPayload });
}
async function describeImageWithModelCore(params) {
return await describeImagesWithModelCore(toImagesDescriptionRequest(params));
}
async function describeImageWithModelPayloadTransformCore(params, onPayload) {
return await describeImagesWithModelPayloadTransformCore(toImagesDescriptionRequest(params), onPayload);
}
//#endregion
export { describeImageWithModelCore, describeImageWithModelPayloadTransformCore, describeImagesWithModelCore, describeImagesWithModelPayloadTransformCore };