UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

577 lines (576 loc) 24.2 kB
import { l as normalizeOptionalString, o as normalizeLowercaseStringOrEmpty, s as normalizeNullableString } from "./string-coerce-CIXf7egm.js"; import { y as uniqueStrings } from "./string-normalization-DsCfAx8q.js"; import { i as createLazyRuntimeNamedExport, r as createLazyRuntimeModule } from "./lazy-runtime-CgCh8H_K.js"; import { n as ok } from "./result-BQGgYouL.js"; import { n as findNormalizedProviderValue } from "./provider-id-DMd-TDFp.js"; import { c as inferUniqueProviderFromConfiguredModels, i as buildModelAliasIndex, y as resolveModelRefFromString } from "./model-selection-shared-BlLyx1r2.js"; import { a as resolveAgentModelPrimaryValue, i as resolveAgentModelFallbackValues } from "./model-input-BuGMCNOz.js"; import { t as resolveDefaultModelForAgent } from "./model-selection-config-BrdmmqKD.js"; import { i as shouldLogVerbose, r as logVerbose } from "./globals-CTaGxEqj.js"; import { i as logWarn } from "./logger-DwECwNVZ.js"; import { l as normalizeMediaFacts } from "./media-facts-8Sl_nPCc.js"; import { r as classifyMediaReferenceSource } from "./media-reference-ZG0ZXjCX.js"; import { a as isProviderAuthError } from "./model-auth-runtime-shared-C48YoQY0.js"; import "./model-selection-di2kjKCB.js"; import { r as mergeInboundPathRoots } from "./inbound-path-policy-DQ5Rksw7.js"; import { i as getDefaultMediaLocalRoots } from "./local-roots-BphuGw-a.js"; import { n as normalizeMediaProviderId, t as normalizeMediaExecutionProviderId } from "./provider-id-DSbuCFIb.js"; import { t as resolveChannelInboundAttachmentRoots } from "./channel-inbound-roots-BiNZ8OaF.js"; import { n as resolveDefaultMediaModelFromRegistry, r as providerSupportsCapability, t as resolveAutoMediaKeyProvidersFromRegistry } from "./provider-registry-metadata-BSy540fQ.js"; import { i as matchesMediaEntryCapability } from "./runtime-media-secret-owner-hrrxUKqA.js"; import { n as buildMediaUnderstandingRegistry, r as getMediaUnderstandingProvider } from "./pdf-extract-D4eXXKMg.js"; import { a as resolveModelEntries, s as resolveScopeDecision } from "./resolve-DyQF7rKc.js"; import { n as resolveOpenAiAudioAuthModelApi } from "./openai-audio-api-CZK5TvIV.js"; import { n as isMinimaxVlmProvider, t as isMinimaxVlmModel } from "./minimax-vlm-BXnJAulK.js"; import { a as normalizeAttachments } from "./attachments.normalize-BdfpE9Qn.js"; import { a as runCliEntry, d as selectAttachments, l as MediaAttachmentCache, o as runProviderEntry, r as formatDecisionSummary, t as buildModelDecision, u as isMediaUnderstandingSkipError } from "./runner.entries-B6FlBFiO.js"; import { r as inspectLocalAudioSelection, t as clearLocalAudioInspectionCacheForTests } from "./local-audio-YEyS3BND.js"; import path from "node:path"; //#region src/media-understanding/runner.attachments.ts /** Normalizes message context media fields for the media-understanding runner. */ function normalizeMediaAttachments(ctx) { const attachments = normalizeAttachments(ctx); return ctx.SkipStickerMediaUnderstanding ? attachments.filter((attachment) => attachment.index !== 0) : attachments; } /** Creates the lazy attachment cache used by image, audio, video, and document providers. */ function createMediaAttachmentCache(attachments, options) { return new MediaAttachmentCache(attachments, options); } //#endregion //#region src/media-understanding/runner.ts const loadHasAvailableAuthForProvider = createLazyRuntimeNamedExport(() => import("./model-auth-RXb_g6lE.js"), "hasAvailableAuthForProvider"); const loadPreparedModelCatalogApi = createLazyRuntimeModule(async () => ({ ...await import("./model-catalog-B9vBAgJt.js"), ...await import("./prepared-model-catalog-BH7NC_p8.js") })); function resolveLiteralProviderApiKey(cfg, providerId) { return normalizeNullableString(findNormalizedProviderValue(cfg?.models?.providers, providerId)?.apiKey); } async function hasProviderAuthAvailable(params) { if (resolveLiteralProviderApiKey(params.cfg, params.provider)) return true; return await (await loadHasAvailableAuthForProvider())({ ...params, modelApi: resolveOpenAiAudioAuthModelApi({ capability: params.capability, providerId: params.provider }) }); } function resolveConfiguredKeyProviderOrder(params) { const configuredProviders = Object.keys(params.cfg.models?.providers ?? {}).map((providerId) => normalizeMediaExecutionProviderId(providerId)).filter(Boolean); const supportedProviders = uniqueStrings(configuredProviders).filter((providerId) => providerSupportsCapability(params.providerRegistry.get(normalizeMediaProviderId(providerId)), params.capability)); return uniqueStrings([...supportedProviders, ...params.fallbackProviders]); } function resolveConfiguredImageModelId(params) { if (isMinimaxVlmProvider(params.providerId)) return; return resolveConfiguredImageModel(params)?.id?.trim() || void 0; } function resolveConfiguredImageModel(params) { return findNormalizedProviderValue(params.cfg.models?.providers, params.providerId)?.models?.find((entry) => { const id = entry?.id?.trim(); return Boolean(id) && entry?.input?.includes("image"); }); } function resolveCatalogImageModelId(params) { const matches = params.catalog.filter((entry) => normalizeMediaProviderId(entry.provider) === normalizeMediaProviderId(params.providerId) && params.modelSupportsVision(entry)); if (matches.length === 0) return; const autoEntry = matches.find((entry) => normalizeLowercaseStringOrEmpty(entry.id) === "auto"); return normalizeOptionalString((autoEntry ?? matches[0])?.id); } async function explicitImageModelVisionStatus(params) { if (isMinimaxVlmProvider(params.providerId) && !isMinimaxVlmModel(params.providerId, params.model)) return "unsupported"; const configured = resolveConfiguredImageModel(params); if (configured?.id?.trim() === params.model && configured.input?.includes("image")) return "supported"; const { findModelInCatalog, loadPreparedModelCatalog, modelSupportsVision } = await loadPreparedModelCatalogApi(); const entry = findModelInCatalog(await loadPreparedModelCatalog({ config: params.cfg, ...params.agentId ? { agentId: params.agentId } : {}, ...params.agentDir ? { agentDir: params.agentDir } : {}, ...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {} }), params.providerId, params.model); if (!entry) return "unknown"; return modelSupportsVision(entry) ? "supported" : "unsupported"; } async function resolveAutoImageModelId(params) { const explicit = normalizeOptionalString(params.explicitModel); if (explicit) { if (await explicitImageModelVisionStatus({ cfg: params.cfg, agentId: params.agentId, providerId: params.providerId, model: explicit, agentDir: params.agentDir, workspaceDir: params.workspaceDir }) !== "unsupported") return explicit; } if (isMinimaxVlmProvider(params.providerId)) return "MiniMax-VL-01"; const configuredModel = resolveConfiguredImageModelId(params); if (configuredModel) return configuredModel; const defaultModel = resolveDefaultMediaModelFromRegistry({ providerId: params.providerId, capability: "image", providerRegistry: params.providerRegistry }); if (defaultModel) return defaultModel; const { resolveDefaultMediaModel } = await import("./defaults-BNwTD3lv.js"); const bundledDefaultModel = resolveDefaultMediaModel({ cfg: params.cfg, providerId: params.providerId, capability: "image", workspaceDir: params.workspaceDir }); if (bundledDefaultModel) return bundledDefaultModel; const { loadPreparedModelCatalog, modelSupportsVision } = await loadPreparedModelCatalogApi(); const catalog = await loadPreparedModelCatalog({ config: params.cfg, ...params.agentId ? { agentId: params.agentId } : {}, ...params.agentDir ? { agentDir: params.agentDir } : {}, ...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {} }); return resolveCatalogImageModelId({ providerId: params.providerId, catalog, modelSupportsVision }); } function buildProviderRegistry(overrides, cfg) { return buildMediaUnderstandingRegistry(overrides, cfg); } function resolveMediaAttachmentLocalRoots(params) { const workspaceDirs = normalizeMediaFacts(params.ctx.media).flatMap((fact) => fact.workspaceDir ? [path.resolve(fact.workspaceDir)] : []); return mergeInboundPathRoots(getDefaultMediaLocalRoots(), workspaceDirs, params.workspaceDir ? [path.resolve(params.workspaceDir)] : void 0, resolveChannelInboundAttachmentRoots(params)); } function clearMediaUnderstandingBinaryCacheForTests() { clearLocalAudioInspectionCacheForTests(); } if (process.env.VITEST || false) globalThis[Symbol.for("openclaw.mediaUnderstandingRunnerTestApi")] = { clearMediaUnderstandingBinaryCacheForTests }; async function resolveKeyEntry(params) { const { cfg, providerRegistry, capability } = params; const checkProvider = (providerId, model) => resolveAutoProviderModelEntry(params, providerId, () => model); const activeProvider = params.activeModel?.provider?.trim(); if (activeProvider) { const activeEntry = await checkProvider(activeProvider, params.activeModel?.model); if (activeEntry) return activeEntry; } for (const providerId of resolveConfiguredKeyProviderOrder({ cfg, providerRegistry, capability, fallbackProviders: resolveAutoMediaKeyProvidersFromRegistry({ capability, providerRegistry }) })) { const entry = await checkProvider(providerId, void 0); if (entry) return entry; } return null; } function resolveImageModelFromAgentDefaults(params) { const refs = []; const primary = resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.imageModel); if (primary?.trim()) refs.push(primary.trim()); for (const fb of resolveAgentModelFallbackValues(params.cfg.agents?.defaults?.imageModel)) if (fb?.trim()) refs.push(fb.trim()); if (refs.length === 0) return []; const defaultProvider = resolveDefaultModelForAgent({ cfg: params.cfg, agentId: params.agentId }).provider; const entries = []; for (const ref of refs) { const effectiveDefaultProvider = ref.includes("/") ? defaultProvider : inferUniqueProviderFromConfiguredModels({ cfg: params.cfg, model: ref, agentId: params.agentId }) ?? defaultProvider; const aliasIndex = buildModelAliasIndex({ cfg: params.cfg, defaultProvider: effectiveDefaultProvider, agentId: params.agentId }); const resolved = resolveModelRefFromString({ cfg: params.cfg, agentId: params.agentId, raw: ref, defaultProvider: effectiveDefaultProvider, aliasIndex }); if (!resolved) continue; entries.push({ type: "provider", provider: resolved.ref.provider, model: resolved.ref.model }); } return entries; } function hasExplicitImageUnderstandingConfig(params) { return (params.cfg.tools?.media?.models ?? []).some((entry) => matchesMediaEntryCapability({ entry, source: "shared", capability: "image", providerRegistry: params.providerRegistry })); } function isMinimaxNativeVisionModel(params) { return isMinimaxVlmProvider(params.provider) && /^MiniMax-M3(\b|[-.])/i.test(params.model?.trim() ?? ""); } async function activeModelSupportsNativeVision(params) { const activeProvider = params.activeModel?.provider?.trim(); if (!activeProvider) return false; if (isMinimaxVlmProvider(activeProvider) && !isMinimaxNativeVisionModel({ provider: activeProvider, model: params.activeModel?.model })) return false; const { findModelInCatalog, loadPreparedModelCatalog, modelSupportsVision } = await loadPreparedModelCatalogApi(); return modelSupportsVision(findModelInCatalog(await loadPreparedModelCatalog({ config: params.cfg, ...params.agentId ? { agentId: params.agentId } : {}, ...params.agentDir ? { agentDir: params.agentDir } : {}, ...params.workspaceDir ? { workspaceDir: params.workspaceDir } : {} }), activeProvider, params.activeModel?.model ?? "")); } async function* resolveAutoAudioEntries(params) { const activeProvider = normalizeMediaExecutionProviderId(params.activeModel?.provider?.trim() ?? ""); const providers = uniqueStrings([...activeProvider ? [activeProvider] : [], ...resolveConfiguredKeyProviderOrder({ ...params, fallbackProviders: resolveAutoMediaKeyProvidersFromRegistry(params) })]); for (const providerId of providers) { const entry = await resolveAutoProviderModelEntry(params, providerId, () => void 0); if (entry) yield { entry }; } const localAudio = await inspectLocalAudioSelection(); for (const entry of localAudio.entries) yield { entry }; } async function resolveAutoEntries(params) { if (params.capability === "image" && !params.nativeVisionActive) { const imageModelEntries = resolveImageModelFromAgentDefaults({ cfg: params.cfg, agentId: params.agentId }); if (imageModelEntries.length > 0) return imageModelEntries.map((entry) => ({ entry })); } const activeEntry = await resolveActiveModelEntry(params); if (activeEntry) return [{ entry: activeEntry }]; const keys = await resolveKeyEntry(params); if (keys) return [{ entry: keys }]; return []; } async function resolveAutoImageModel(params) { const providerRegistry = buildProviderRegistry(void 0, params.cfg); const entries = await resolveAutoEntries({ ...params, providerRegistry, capability: "image", nativeVisionActive: false }); for (const { entry } of entries) { if (entry.type === "cli") continue; const provider = entry.provider; const model = entry.model?.trim(); if (provider && model) return { provider, model }; } return null; } async function resolveActiveModelEntry(params) { const activeProviderRaw = params.activeModel?.provider?.trim(); if (!activeProviderRaw) return null; const providerId = normalizeMediaExecutionProviderId(activeProviderRaw); if (!providerId) return null; return await resolveAutoProviderModelEntry(params, providerId, () => params.activeModel?.model); } async function resolveAutoProviderModelEntry(params, providerId, readModel) { const provider = getMediaUnderstandingProvider(providerId, params.providerRegistry); if (!providerSupportsCapability(provider, params.capability)) return null; if (!(params.capability === "audio" && provider?.transcribeAudioWithContext) && !await hasProviderAuthAvailable({ capability: params.capability, provider: providerId, cfg: params.cfg, agentDir: params.agentDir, workspaceDir: params.workspaceDir })) return null; let model; if (params.capability === "image") model = await resolveAutoImageModelId({ cfg: params.cfg, agentId: params.agentId, providerId, providerRegistry: params.providerRegistry, explicitModel: readModel(), agentDir: params.agentDir, workspaceDir: params.workspaceDir }); else if (params.capability === "audio") model = resolveDefaultMediaModelFromRegistry({ providerId, capability: "audio", providerRegistry: params.providerRegistry }); else model = readModel() ?? resolveDefaultMediaModelFromRegistry({ providerId, capability: "video", providerRegistry: params.providerRegistry }); if (params.capability === "image" && !model) return null; return { type: "provider", provider: providerId, model }; } async function runAttachmentEntries(params) { const { entries, capability } = params; const attachmentIndex = params.attachment.index; const attempts = []; for await (const candidate of entries) { const { entry } = candidate; const entryType = entry.type ?? (entry.command ? "cli" : "provider"); try { const attempt = entryType === "cli" ? ok(await runCliEntry({ capability, entry, cfg: params.cfg, ctx: params.ctx, attachment: params.attachment, cache: params.cache, config: params.config })) : await runProviderEntry({ capability, entry, cfg: params.cfg, ctx: params.ctx, attachmentIndex, cache: params.cache, agentId: params.agentId, agentDir: params.agentDir, workspaceDir: params.workspaceDir, providerRegistry: params.providerRegistry, config: params.config, secretOwnerId: candidate.secretOwnerId }); if (!attempt.ok) { if (!(params.automaticAudio && isProviderAuthError(attempt.error, "missing-provider-auth"))) attempts.push(buildModelDecision({ entry, entryType, outcome: "failed", reason: String(attempt.error) })); continue; } const result = attempt.value; if (result?.text) { const decision = buildModelDecision({ entry, entryType, outcome: "success" }); if (result.provider) decision.provider = result.provider; decision.model = result.model; if (result.requestedBackend) decision.requestedBackend = result.requestedBackend; if (result.observedBackend) decision.observedBackend = result.observedBackend; attempts.push(decision); return { output: result, attempts }; } attempts.push(buildModelDecision({ entry, entryType, outcome: "skipped", reason: "empty output" })); } catch (err) { if (isMediaUnderstandingSkipError(err)) { attempts.push(buildModelDecision({ entry, entryType, outcome: "skipped", reason: `${err.reason}: ${err.message}` })); if (shouldLogVerbose()) logVerbose(`Skipping ${capability} model due to ${err.reason}: ${err.message}`); } else { attempts.push(buildModelDecision({ entry, entryType, outcome: "failed", reason: String(err) })); if (shouldLogVerbose()) logVerbose(`${capability} understanding failed: ${String(err)}`); } } if (params.automaticAudio && entryType === "provider") break; } return { output: null, attempts }; } function hasFailedMediaAttempt(attachments) { return attachments.some((attachment) => attachment.attempts.some((attempt) => attempt.outcome === "failed")); } function createAttachmentDispositions(indexes, disposition) { return Object.fromEntries(indexes.map((index) => [index, disposition])); } async function runCapability(params) { const { capability, cfg, ctx } = params; const config = params.config ?? cfg.tools?.media?.[capability] ?? {}; const selection = selectAttachments({ capability, attachments: params.media, policy: config.attachments }); const selectedAttachmentIndexes = selection.selected.map((attachment) => attachment.index); const activeProvider = params.activeModel?.provider?.trim(); let nativeVisionProbe; const resolveNativeVisionFlag = () => { nativeVisionProbe ??= activeModelSupportsNativeVision({ cfg, agentId: params.agentId, activeModel: params.activeModel, agentDir: params.agentDir, workspaceDir: params.workspaceDir }).catch((err) => { if (shouldLogVerbose()) logVerbose(`native vision support probe failed: ${String(err)}`); }); return nativeVisionProbe; }; const buildDispositions = (selectedDisposition, droppedDisposition = selectedDisposition) => ({ ...createAttachmentDispositions(selectedAttachmentIndexes, selectedDisposition), ...createAttachmentDispositions(selection.droppedAttachmentIndexes, droppedDisposition) }); const rendersMarker = (dispositions) => Object.values(dispositions).some((d) => d.kind !== "handled" && d.kind !== "handed-to-native-vision"); const buildDecision = async (outcome, attachments, attachmentDispositions) => { const nativeVisionActive = capability === "image" && (nativeVisionProbe !== void 0 || rendersMarker(attachmentDispositions)) ? await resolveNativeVisionFlag() : void 0; return { capability, outcome, attachments, attachmentDispositions, ...nativeVisionActive !== void 0 ? { nativeVisionActive } : {} }; }; if (config?.enabled === false) return { outputs: [], decision: await buildDecision("disabled", [], buildDispositions({ kind: "capability-disabled" })) }; if (selection.selected.length === 0) return { outputs: [], decision: await buildDecision("no-attachment", [], {}) }; if (resolveScopeDecision({ scope: config?.scope, ctx }) === "deny") { if (shouldLogVerbose()) logVerbose(`${capability} understanding disabled by scope policy.`); return { outputs: [], decision: await buildDecision("scope-deny", selection.selected.map((item) => ({ attachmentIndex: item.index, attempts: [] })), buildDispositions({ kind: "scope-denied" })) }; } if (capability === "image" && activeProvider && !hasExplicitImageUnderstandingConfig({ cfg, providerRegistry: params.providerRegistry }) && await resolveNativeVisionFlag() === true) { if (shouldLogVerbose()) logVerbose("Skipping image understanding: primary model supports vision natively"); const attempt = { type: "provider", provider: activeProvider, model: params.activeModel?.model?.trim() || void 0, outcome: "skipped", reason: "primary model supports vision natively" }; const nativeDeliverable = (item) => Boolean(item.path) || Boolean(item.url) && classifyMediaReferenceSource(item.url ?? "").isMediaStoreUrl; const attachmentDispositions = buildDispositions({ kind: "handed-to-native-vision" }, { kind: "not-selected" }); for (const item of params.media) if (attachmentDispositions[item.index] && !nativeDeliverable(item)) attachmentDispositions[item.index] = { kind: "failed", reason: "remote-url image is not natively deliverable" }; return { outputs: [], decision: await buildDecision("skipped", selection.selected.map((item) => nativeDeliverable(item) ? { attachmentIndex: item.index, attempts: [attempt], chosen: attempt } : { attachmentIndex: item.index, attempts: [] }), attachmentDispositions) }; } const entries = resolveModelEntries({ cfg, capability, config, providerRegistry: params.providerRegistry }); const automaticAudio = capability === "audio" && entries.length === 0; let resolvedEntries = entries; if (!automaticAudio && resolvedEntries.length === 0) resolvedEntries = await resolveAutoEntries({ cfg, agentId: params.agentId, agentDir: params.agentDir, workspaceDir: params.workspaceDir, providerRegistry: params.providerRegistry, capability, activeModel: params.activeModel, config, nativeVisionActive: capability === "image" && await resolveNativeVisionFlag() === true }); if (!automaticAudio && resolvedEntries.length === 0) return { outputs: [], decision: await buildDecision("skipped", selection.selected.map((item) => ({ attachmentIndex: item.index, attempts: [] })), buildDispositions({ kind: "no-model" }, { kind: "not-selected" })) }; const outputs = []; const attachmentDecisions = []; const attachmentDispositions = buildDispositions({ kind: "failed" }, { kind: "not-selected" }); for (const attachment of selection.selected) { const { output, attempts } = await runAttachmentEntries({ capability, cfg, ctx, attachment, agentId: params.agentId, agentDir: params.agentDir, workspaceDir: params.workspaceDir, providerRegistry: params.providerRegistry, cache: params.attachments, entries: automaticAudio ? resolveAutoAudioEntries({ cfg, agentId: params.agentId, agentDir: params.agentDir, workspaceDir: params.workspaceDir, providerRegistry: params.providerRegistry, capability, activeModel: params.activeModel, nativeVisionActive: false }) : resolvedEntries, automaticAudio, config }); if (output) outputs.push(output); attachmentDispositions[attachment.index] = output ? { kind: "handled" } : attempts.length > 0 ? { kind: "failed" } : { kind: "no-model" }; attachmentDecisions.push({ attachmentIndex: attachment.index, attempts, chosen: attempts.find((attempt) => attempt.outcome === "success") }); } const decision = await buildDecision(outputs.length > 0 ? "success" : hasFailedMediaAttempt(attachmentDecisions) ? "failed" : "skipped", attachmentDecisions, attachmentDispositions); if (decision.outcome === "failed") logWarn(`media-understanding: ${formatDecisionSummary(decision)}`); else if (shouldLogVerbose()) logVerbose(`Media understanding ${formatDecisionSummary(decision)}`); return { outputs, decision }; } //#endregion export { createMediaAttachmentCache as a, runCapability as i, resolveAutoImageModel as n, normalizeMediaAttachments as o, resolveMediaAttachmentLocalRoots as r, buildProviderRegistry as t };