openclaw/dist/extensions/google/media-understanding-provider.js

Multi-channel AI gateway with extensible messaging integrations

import { m as readProviderJsonResponse, r as assertOkOrThrowProviderError } from "../../provider-http-errors-U-nhuk_f.js";
import { l as postJsonRequest } from "../../shared-DCDpYrmy.js";
import "../../provider-http-k9RMI7iG.js";
import { n as normalizeGoogleModelId } from "../../model-id-CAmKILzd.js";
import { s as DEFAULT_GOOGLE_API_BASE_URL } from "../../provider-policy-BjaZN4xs.js";
import { t as resolveGoogleGenerativeAiHttpRequestConfig } from "../../api-BIkqybAZ.js";
import "../../runtime-api-B4NLEMi7.js";
//#region extensions/google/media-understanding-provider.ts
const DEFAULT_GOOGLE_AUDIO_MODEL = "gemini-3-flash-preview";
const DEFAULT_GOOGLE_VIDEO_MODEL = "gemini-3-flash-preview";
const DEFAULT_GOOGLE_AUDIO_PROMPT = "Transcribe the audio.";
const DEFAULT_GOOGLE_VIDEO_PROMPT = "Describe the video.";
async function generateGeminiInlineDataText(params) {
	const fetchFn = params.fetchFn ?? fetch;
	const model = (() => {
		const trimmed = params.model?.trim();
		if (!trimmed) return params.defaultModel;
		return normalizeGoogleModelId(trimmed);
	})();
	const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveGoogleGenerativeAiHttpRequestConfig({
		apiKey: params.apiKey,
		baseUrl: params.baseUrl,
		headers: params.headers,
		request: params.request,
		capability: params.defaultMime.startsWith("audio/") ? "audio" : "video",
		transport: "media-understanding"
	});
	const url = `${baseUrl ?? params.defaultBaseUrl}/models/${model}:generateContent`;
	const body = { contents: [{
		role: "user",
		parts: [{ text: (() => {
			return params.prompt?.trim() || params.defaultPrompt;
		})() }, { inline_data: {
			mime_type: params.mime ?? params.defaultMime,
			data: params.buffer.toString("base64")
		} }]
	}] };
	const { response: res, release } = await postJsonRequest({
		url,
		headers,
		body,
		timeoutMs: params.timeoutMs,
		...params.signal ? { signal: params.signal } : {},
		fetchFn,
		allowPrivateNetwork,
		dispatcherPolicy
	});
	try {
		await assertOkOrThrowProviderError(res, params.httpErrorLabel);
		const text = ((await readProviderJsonResponse(res, params.httpErrorLabel)).candidates?.[0]?.content?.parts ?? []).map((part) => part?.text?.trim()).filter(Boolean).join("\n");
		if (!text) throw new Error(params.missingTextError);
		return {
			text,
			model
		};
	} finally {
		await release();
	}
}
async function transcribeGeminiAudio(params) {
	const { text, model } = await generateGeminiInlineDataText({
		...params,
		defaultBaseUrl: DEFAULT_GOOGLE_API_BASE_URL,
		defaultModel: DEFAULT_GOOGLE_AUDIO_MODEL,
		defaultPrompt: DEFAULT_GOOGLE_AUDIO_PROMPT,
		defaultMime: "audio/wav",
		httpErrorLabel: "Audio transcription failed",
		missingTextError: "Audio transcription response missing text"
	});
	return {
		text,
		model
	};
}
async function describeGeminiVideo(params) {
	const { text, model } = await generateGeminiInlineDataText({
		...params,
		defaultBaseUrl: DEFAULT_GOOGLE_API_BASE_URL,
		defaultModel: DEFAULT_GOOGLE_VIDEO_MODEL,
		defaultPrompt: DEFAULT_GOOGLE_VIDEO_PROMPT,
		defaultMime: "video/mp4",
		httpErrorLabel: "Video description failed",
		missingTextError: "Video description response missing text"
	});
	return {
		text,
		model
	};
}
const googleMediaUnderstandingProvider = {
	id: "google",
	capabilities: [
		"image",
		"audio",
		"video"
	],
	defaultModels: {
		image: DEFAULT_GOOGLE_VIDEO_MODEL,
		audio: DEFAULT_GOOGLE_AUDIO_MODEL,
		video: DEFAULT_GOOGLE_VIDEO_MODEL
	},
	autoPriority: {
		image: 30,
		audio: 40,
		video: 10
	},
	nativeDocumentInputs: ["pdf"],
	describeImage: void 0,
	describeImages: void 0,
	transcribeAudio: transcribeGeminiAudio,
	describeVideo: describeGeminiVideo
};
//#endregion
export { describeGeminiVideo, googleMediaUnderstandingProvider, transcribeGeminiAudio };