@ai-sdk/google-vertex
Version:
2,031 lines • 71 kB
JavaScript
import { WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE, combineHeaders, connectToWebSocket, convertBase64ToUint8Array, convertToBase64, convertUint8ArrayToBase64, createJsonErrorResponseHandler, createJsonResponseHandler, generateId, getRuntimeEnvironmentUserAgent, isValidHostnamePart, lazySchema, loadOptionalSetting, loadSetting, normalizeHeaders, parseProviderOptions, postJsonToApi, resolve, safeParseJSON, serializeModelOptions, waitForWebSocketBufferDrain, withUserAgentSuffix, withoutTrailingSlash, zodSchema } from "@ai-sdk/provider-utils";
import { GoogleInteractionsLanguageModel, GoogleLanguageModel, GoogleSpeechModel, getGoogleLanguageModelEntry, googleProviderOptions, googleTools } from "@ai-sdk/google/internal";
import { AISDKError, InvalidArgumentError, LoadAPIKeyError, TooManyEmbeddingValuesForCallError } from "@ai-sdk/provider";
import { z } from "zod/v4";
//#region src/google-vertex-model-catalog-config.ts
function requestConfig(modelId) {
const embedContent = modelId === "gemini-embedding-2" || modelId === "gemini-embedding-2-preview";
return {
language: {
endpoint: modelId.startsWith("endpoints/"),
maxOutputTokens: modelId === "meta/llama-4-maverick-17b-128e-instruct-maas" || modelId === "meta/llama-4-scout-17b-16e-instruct-maas" ? 8192 : void 0
},
image: { gemini: modelId.startsWith("gemini-") },
embedding: {
endpoint: embedContent ? "embedContent" : "predict",
maxEmbeddingsPerCall: embedContent ? 1 : 250
},
speech: { api: modelId.startsWith("chirp") ? "cloud-tts" : "gemini" },
transcription: {
api: modelId.startsWith("gemini") ? "gemini" : "cloud-speech",
live: modelId.includes("-live")
}
};
}
function getGoogleVertexRequestConfig(modelId) {
return requestConfig(modelId);
}
//#endregion
//#region src/version.ts
const VERSION = "5.0.141";
//#endregion
//#region src/google-vertex-error.ts
const googleVertexErrorDataSchema = lazySchema(() => zodSchema(z.object({ error: z.object({
code: z.number().nullable(),
message: z.string(),
status: z.string()
}) })));
const googleVertexFailedResponseHandler = createJsonErrorResponseHandler({
errorSchema: googleVertexErrorDataSchema,
errorToMessage: (data) => data.error.message
});
//#endregion
//#region src/google-vertex-embedding-model-options.ts
const googleVertexEmbeddingModelOptions = lazySchema(() => zodSchema(z.object({
/**
* Optional. Optional reduced dimension for the output embedding.
* If set, excessive values in the output embedding are truncated from the end.
*/
outputDimensionality: z.number().optional(),
/**
* Optional. Specifies the task type for generating embeddings.
* Supported task types:
* - SEMANTIC_SIMILARITY: Optimized for text similarity.
* - CLASSIFICATION: Optimized for text classification.
* - CLUSTERING: Optimized for clustering texts based on similarity.
* - RETRIEVAL_DOCUMENT: Optimized for document retrieval.
* - RETRIEVAL_QUERY: Optimized for query-based retrieval.
* - QUESTION_ANSWERING: Optimized for answering questions.
* - FACT_VERIFICATION: Optimized for verifying factual information.
* - CODE_RETRIEVAL_QUERY: Optimized for retrieving code blocks based on natural language queries.
*/
taskType: z.enum([
"SEMANTIC_SIMILARITY",
"CLASSIFICATION",
"CLUSTERING",
"RETRIEVAL_DOCUMENT",
"RETRIEVAL_QUERY",
"QUESTION_ANSWERING",
"FACT_VERIFICATION",
"CODE_RETRIEVAL_QUERY"
]).optional(),
/**
* Optional. The title of the document being embedded.
* Only valid when task_type is set to 'RETRIEVAL_DOCUMENT'.
* Helps the model produce better embeddings by providing additional context.
*/
title: z.string().optional(),
/**
* Optional. When set to true, input text will be truncated. When set to false,
* an error is returned if the input text is longer than the maximum length supported by the model. Defaults to true.
*/
autoTruncate: z.boolean().optional()
})));
//#endregion
//#region src/google-vertex-embedding-model.ts
var GoogleVertexEmbeddingModel = class GoogleVertexEmbeddingModel {
static [WORKFLOW_SERIALIZE](model) {
return serializeModelOptions({
modelId: model.modelId,
config: model.config
});
}
static [WORKFLOW_DESERIALIZE](options) {
return new GoogleVertexEmbeddingModel(options.modelId, options.config);
}
get provider() {
return this.config.provider;
}
get maxEmbeddingsPerCall() {
return usesEmbedContentEndpoint(this.modelId) ? 1 : 250;
}
constructor(modelId, config) {
this.specificationVersion = "v4";
this.supportsParallelCalls = true;
this.modelId = modelId;
this.config = config;
}
async doEmbed({ values, dimensions, headers, abortSignal, providerOptions }) {
let googleOptions = await parseProviderOptions({
provider: "googleVertex",
providerOptions,
schema: googleVertexEmbeddingModelOptions
});
if (googleOptions == null) googleOptions = await parseProviderOptions({
provider: "vertex",
providerOptions,
schema: googleVertexEmbeddingModelOptions
});
if (googleOptions == null) googleOptions = await parseProviderOptions({
provider: "google",
providerOptions,
schema: googleVertexEmbeddingModelOptions
});
googleOptions = googleOptions ?? {};
if (values.length > this.maxEmbeddingsPerCall) throw new TooManyEmbeddingValuesForCallError({
provider: this.provider,
modelId: this.modelId,
maxEmbeddingsPerCall: this.maxEmbeddingsPerCall,
values
});
const mergedHeaders = combineHeaders(this.config.headers ? await resolve(this.config.headers) : void 0, headers);
if (usesEmbedContentEndpoint(this.modelId)) {
const { responseHeaders, value: response, rawValue } = await postJsonToApi({
url: `${this.config.baseURL}/models/${this.modelId}:embedContent`,
headers: mergedHeaders,
body: {
content: { parts: [{ text: values[0] }] },
embedContentConfig: {
outputDimensionality: googleOptions.outputDimensionality ?? dimensions,
taskType: googleOptions.taskType,
title: googleOptions.title,
autoTruncate: googleOptions.autoTruncate
}
},
failedResponseHandler: googleVertexFailedResponseHandler,
successfulResponseHandler: createJsonResponseHandler(googleVertexEmbedContentResponseSchema),
abortSignal,
fetch: this.config.fetch
});
return {
warnings: [],
embeddings: [response.embedding.values],
usage: response.usageMetadata?.promptTokenCount == null ? void 0 : { tokens: response.usageMetadata.promptTokenCount },
response: {
headers: responseHeaders,
body: rawValue
}
};
}
const url = `${this.config.baseURL}/models/${this.modelId}:predict`;
const { responseHeaders, value: response, rawValue } = await postJsonToApi({
url,
headers: mergedHeaders,
body: {
instances: values.map((value) => ({
content: value,
task_type: googleOptions.taskType,
title: googleOptions.title
})),
parameters: {
outputDimensionality: googleOptions.outputDimensionality ?? dimensions,
autoTruncate: googleOptions.autoTruncate
}
},
failedResponseHandler: googleVertexFailedResponseHandler,
successfulResponseHandler: createJsonResponseHandler(googleVertexTextEmbeddingResponseSchema),
abortSignal,
fetch: this.config.fetch
});
return {
warnings: [],
embeddings: response.predictions.map((prediction) => prediction.embeddings.values),
usage: { tokens: response.predictions.reduce((tokenCount, prediction) => tokenCount + prediction.embeddings.statistics.token_count, 0) },
response: {
headers: responseHeaders,
body: rawValue
}
};
}
};
const googleVertexTextEmbeddingResponseSchema = lazySchema(() => zodSchema(z.object({ predictions: z.array(z.object({ embeddings: z.object({
values: z.array(z.number()),
statistics: z.object({ token_count: z.number() })
}) })) })));
const googleVertexEmbedContentResponseSchema = lazySchema(() => zodSchema(z.object({
embedding: z.object({ values: z.array(z.number()) }),
usageMetadata: z.object({ promptTokenCount: z.number().nullish() }).nullish()
})));
function usesEmbedContentEndpoint(modelId) {
return getGoogleVertexRequestConfig(modelId).embedding?.endpoint === "embedContent";
}
//#endregion
//#region src/google-vertex-model-catalog.ts
const documentedImageDetails = {
"gemini-nano-banana-2.1": {
imageGeneration: true,
aspectRatios: [
"1:1",
"2:3",
"3:2",
"3:4",
"4:3",
"4:5",
"5:4",
"9:16",
"16:9",
"21:9",
"1:8",
"8:1",
"1:4",
"4:1"
]
},
"gemini-3.1-flash-image-preview": {
imageGeneration: true,
aspectRatios: [
"1:1",
"2:3",
"3:2",
"3:4",
"4:3",
"4:5",
"5:4",
"9:16",
"16:9",
"21:9"
]
},
"gemini-3-pro-image-preview": {
imageGeneration: true,
aspectRatios: [
"1:1",
"2:3",
"3:2",
"3:4",
"4:3",
"4:5",
"5:4",
"9:16",
"16:9",
"21:9"
]
},
"gemini-2.5-flash-image": {
imageGeneration: true,
aspectRatios: [
"1:1",
"2:3",
"3:2",
"3:4",
"4:3",
"4:5",
"5:4",
"9:16",
"16:9",
"21:9"
]
}
};
const documentedTranscriptionDetails = {
chirp_2: {
wordTimestampDescription: "Available with a potential quality and speed tradeoff",
languageDetectionDescription: "Auto detection with `['auto']`"
},
chirp_3: {
wordTimestampDescription: "Available with a potential transcription quality tradeoff",
languageDetectionDescription: "Auto detection with `['auto']`"
},
telephony: {
wordTimestampDescription: "Available",
languageDetectionDescription: "Explicit supported language codes, with alternative language detection support"
}
};
const googleVertexProviderOptions = {
language: googleProviderOptions.language,
image: Object.fromEntries(Object.entries(googleProviderOptions.language).filter(([key]) => key !== "responseModalities")),
embedding: {
outputDimensionality: { description: "Reduced output dimensions." },
taskType: { description: "Task for which embeddings are optimized." },
title: { description: "Document title; requires RETRIEVAL_DOCUMENT." },
autoTruncate: { description: "Truncate inputs exceeding the model context." }
},
speech: {
speechMetadata: { description: "Directions for Gemini 3.8 speech." },
turns: { description: "Structured transcript for Gemini 3.8 speech." },
multiSpeakerVoiceConfig: { description: "Voice configuration for dialogue speakers." }
},
transcription: {
languageCodes: { description: "Language hints or automatic language detection." },
enableAutomaticPunctuation: { description: "Include punctuation in Cloud Speech transcripts." },
enableWordTimeOffsets: { description: "Include Cloud Speech word timestamps." },
region: { description: "Cloud Speech region." },
customVocabulary: { description: "Vocabulary hints for Gemini transcription." },
wordTimestamp: { description: "Include Gemini word timestamps." },
diarization: { description: "Enable Gemini speaker diarization." },
mode: { description: "Verbatim or smart Gemini transcription formatting." }
},
video: {
pollIntervalMs: { description: "Polling interval." },
pollTimeoutMs: { description: "Polling timeout." },
personGeneration: { description: "Policy for generating people." },
negativePrompt: { description: "Content to avoid." },
generateAudio: { description: "Generate synchronized audio." },
gcsOutputDirectory: { description: "Cloud Storage output directory." },
referenceImages: { description: "Reference images for video generation." }
}
};
function languageEntry(modelId) {
const entry = getGoogleLanguageModelEntry(modelId);
return {
capabilities: { language: entry.capabilities.language },
providerOptions: { language: {
...entry.providerOptions.language,
serviceTier: false
} },
requestConfig: { language: getGoogleVertexRequestConfig(modelId).language }
};
}
function getGoogleVertexImageModelEntry(modelId) {
const known = imageModelIds.some((id) => id === modelId);
return {
capabilities: { image: {
sizes: { description: "Use aspectRatio; the size option is unsupported." },
fileInput: known ? true : "unknown",
maskInput: known ? false : "unknown",
maxImagesPerCall: 1,
...documentedImageDetails[modelId]
} },
providerOptions: { image: Object.fromEntries(Object.keys(googleVertexProviderOptions.image).map((key) => [key, key === "serviceTier" ? false : { condition: "Support follows the Gemini language model and image configuration." }])) },
requestConfig: { image: getGoogleVertexRequestConfig(modelId).image }
};
}
function embeddingEntry(modelId) {
return {
capabilities: { embedding: {
defaultDimensions: modelId.startsWith("gemini-embedding-") ? 3072 : 768,
customDimensions: "unknown",
maxEmbeddingsPerCall: getGoogleVertexRequestConfig(modelId).embedding.maxEmbeddingsPerCall,
supportsParallelCalls: true,
multimodal: getGoogleVertexRequestConfig(modelId).embedding.endpoint === "embedContent"
} },
providerOptions: { embedding: {
outputDimensionality: "unknown",
taskType: "unknown",
title: { condition: "Requires RETRIEVAL_DOCUMENT." },
autoTruncate: true
} },
requestConfig: { embedding: getGoogleVertexRequestConfig(modelId).embedding }
};
}
function speechEntry(modelId) {
const chirp = getGoogleVertexRequestConfig(modelId).speech?.api === "cloud-tts";
return {
capabilities: { speech: {
instructions: !chirp,
multiSpeaker: !chirp
} },
providerOptions: { speech: {
multiSpeakerVoiceConfig: chirp ? false : true,
speechMetadata: chirp ? false : { condition: "Requires Gemini 3.8 TTS." },
turns: chirp ? false : { condition: "Requires Gemini 3.8 TTS." }
} },
requestConfig: { speech: getGoogleVertexRequestConfig(modelId).speech }
};
}
function transcriptionEntry(modelId) {
const config = getGoogleVertexRequestConfig(modelId).transcription;
const gemini = config.api === "gemini";
return {
capabilities: { transcription: {
transcription: !config.live,
streaming: config.live,
duration: gemini ? false : true,
segments: config.live ? false : { condition: "Requires word timestamps." },
language: true,
...documentedTranscriptionDetails[modelId]
} },
providerOptions: { transcription: {
languageCodes: true,
enableAutomaticPunctuation: !gemini,
enableWordTimeOffsets: !gemini,
region: !gemini,
customVocabulary: gemini,
wordTimestamp: gemini,
diarization: gemini,
mode: gemini
} },
requestConfig: { transcription: config }
};
}
function videoEntry(modelId) {
const veo3 = modelId.startsWith("veo-3");
return {
capabilities: { video: {
inputs: {
text: true,
image: true,
firstFrame: true,
lastFrame: { condition: "Requires a Veo model supporting frame interpolation." },
referenceImage: { condition: "Requires a Veo model supporting reference images." },
referenceVideo: false
},
generateAudio: veo3,
aspectRatio: true,
resolution: true,
duration: true,
fps: false,
seed: true,
maxVideosPerCall: 4
} },
providerOptions: { video: {
pollIntervalMs: true,
pollTimeoutMs: true,
personGeneration: true,
negativePrompt: true,
generateAudio: veo3,
gcsOutputDirectory: true,
referenceImages: { condition: "Requires a Veo model supporting reference images." }
} }
};
}
const languageModelIds = [
"gemini-nano-banana-2.1",
"gemini-4-argon",
"gemini-4-carbon",
"gemini-4-hydrogen",
"gemini-3.8-flash",
"gemini-3.7-flash",
"gemini-3.6-flash",
"gemini-3.5-flash",
"gemini-3.5-flash-lite",
"gemini-2.5-pro",
"gemini-2.5-flash",
"gemini-2.5-flash-lite",
"gemini-2.0-flash-lite",
"gemini-2.0-flash",
"gemini-2.0-flash-001",
"gemini-1.5-flash",
"gemini-1.5-flash-001",
"gemini-1.5-flash-002",
"gemini-1.5-pro",
"gemini-1.5-pro-001",
"gemini-1.5-pro-002",
"gemini-1.0-pro-001",
"gemini-1.0-pro-vision-001",
"gemini-1.0-pro",
"gemini-1.0-pro-002",
"gemini-2.0-flash-lite-preview-02-05",
"gemini-2.5-flash-preview-09-2025",
"gemini-3-pro-preview",
"gemini-3-pro-image-preview",
"gemini-3-flash-preview",
"gemini-3.1-pro-preview",
"gemini-3.1-flash-image-preview",
"gemini-3.1-flash-lite-preview",
"gemini-2.0-pro-exp-02-05",
"gemini-2.0-flash-exp"
];
const imageModelIds = [
"gemini-2.5-flash-image",
"gemini-3-pro-image-preview",
"gemini-3.1-flash-image-preview",
"gemini-nano-banana-2.1"
];
const embeddingModelIds = [
"textembedding-gecko",
"textembedding-gecko@001",
"textembedding-gecko@003",
"textembedding-gecko-multilingual",
"textembedding-gecko-multilingual@001",
"text-multilingual-embedding-002",
"text-embedding-004",
"text-embedding-005",
"gemini-embedding-001",
"gemini-embedding-2",
"gemini-embedding-2-preview"
];
const videoModelIds = [
"veo-2.0-generate-preview",
"veo-2.0-generate-exp",
"veo-2.0-generate-001",
"veo-3.0-generate-001",
"veo-3.0-fast-generate-001",
"veo-3.0-generate-preview",
"veo-3.0-fast-generate-preview",
"veo-3.1-generate-001",
"veo-3.1-fast-generate-001",
"veo-3.1-generate-preview",
"veo-3.1-fast-generate-preview"
];
const speechModelIds = [
"gemini-2.5-flash-tts",
"gemini-2.5-pro-tts",
"gemini-2.5-flash-lite-preview-tts",
"gemini-3.1-flash-tts-preview",
"chirp-3-hd"
];
const transcriptionModelIds = [
"chirp_2",
"chirp_3",
"telephony"
];
const geminiTranscriptionModelIds = ["gemini-3.5-transcribe", "gemini-3.5-transcribe-live"];
[
...languageModelIds.map((id) => [id, languageEntry(id)]),
...imageModelIds.map((id) => [id, getGoogleVertexImageModelEntry(id)]),
...embeddingModelIds.map((id) => [id, embeddingEntry(id)]),
...videoModelIds.map((id) => [id, videoEntry(id)]),
...speechModelIds.map((id) => [id, speechEntry(id)]),
...transcriptionModelIds.map((id) => [id, transcriptionEntry(id)]),
...geminiTranscriptionModelIds.map((id) => [id, transcriptionEntry(id)])
].reduce((catalog, [id, entry]) => {
const previous = catalog[id];
catalog[id] = {
capabilities: {
...previous?.capabilities,
...entry.capabilities
},
providerOptions: {
...previous?.providerOptions,
...entry.providerOptions
},
requestConfig: {
...previous?.requestConfig,
...entry.requestConfig
}
};
return catalog;
}, {});
Object.fromEntries(Object.entries({
"claude-fable-5-1": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: true
},
"claude-fable-5-2": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: true
},
"claude-3-7-sonnet@20250219": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: true
},
"claude-3-5-sonnet-v2@20241022": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: true
},
"claude-3-5-sonnet@20240620": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: false
},
"claude-3-5-haiku@20241022": {
image: false,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: false
},
"claude-3-sonnet@20240229": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: false
},
"claude-3-haiku@20240307": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: false
},
"claude-3-opus@20240229": {
image: true,
objectGeneration: true,
tools: true,
streaming: true,
computerUse: false
},
"claude-fable-5": void 0,
"claude-haiku-5-5": void 0,
"claude-opus-4-1@20250805": void 0,
"claude-opus-4-5@20251101": void 0,
"claude-opus-4-6": void 0,
"claude-opus-4-7": void 0,
"claude-opus-4-8": void 0,
"claude-opus-4@20250514": void 0,
"claude-opus-5": void 0,
"claude-opus-5-5": void 0,
"claude-sonnet-4-5@20250929": void 0,
"claude-sonnet-4-6": void 0,
"claude-sonnet-4@20250514": void 0,
"claude-sonnet-5": void 0,
"claude-sonnet-5-5": void 0
}).flatMap(([id, support]) => {
if (support == null) return [];
return [[id, { capabilities: { language: {
inputs: {
image: support.image,
audio: false
},
objectGeneration: support.objectGeneration,
tools: {
usage: support.tools,
streaming: support.streaming,
computerUse: support.computerUse
},
reasoning: { support: "unknown" },
sampling: {
temperature: "unknown",
topP: "unknown",
logprobs: false
}
} } }]];
}));
Object.fromEntries(Object.entries({
"xai/grok-4.20-reasoning": { reasoning: true },
"xai/grok-4.20-non-reasoning": { reasoning: false },
"xai/grok-4.1-fast-reasoning": { reasoning: true },
"xai/grok-4.1-fast-non-reasoning": { reasoning: false }
}).map(([id, support]) => {
return [id, { capabilities: { language: {
inputs: {
image: "unknown",
audio: "unknown"
},
objectGeneration: "unknown",
tools: {
usage: "unknown",
streaming: "unknown"
},
reasoning: { support: support.reasoning },
sampling: {
temperature: "unknown",
topP: "unknown",
logprobs: "unknown"
}
} } }];
}));
Object.fromEntries(Object.entries({
"deepseek-ai/deepseek-r1-0528-maas": { publisher: "DeepSeek" },
"deepseek-ai/deepseek-v3.1-maas": { publisher: "DeepSeek" },
"deepseek-ai/deepseek-v3.2-maas": { publisher: "DeepSeek" },
"openai/gpt-oss-120b-maas": { publisher: "OpenAI" },
"openai/gpt-oss-20b-maas": { publisher: "OpenAI" },
"meta/llama-4-maverick-17b-128e-instruct-maas": { publisher: "Meta" },
"meta/llama-4-scout-17b-16e-instruct-maas": { publisher: "Meta" },
"minimax/minimax-m2-maas": { publisher: "MiniMax" },
"qwen/qwen3-coder-480b-a35b-instruct-maas": { publisher: "Qwen" },
"qwen/qwen3-next-80b-a3b-instruct-maas": { publisher: "Qwen" },
"qwen/qwen3-next-80b-a3b-thinking-maas": { publisher: "Qwen" },
"moonshotai/kimi-k2-thinking-maas": { publisher: "Moonshot" }
}).map(([id, support]) => {
return [id, {
capabilities: { language: {
inputs: {
image: "unknown",
audio: "unknown"
},
objectGeneration: "unknown",
tools: {
usage: "unknown",
streaming: "unknown"
},
reasoning: { support: "unknown" },
sampling: {
temperature: "unknown",
topP: "unknown",
logprobs: "unknown"
},
publisher: support.publisher
} },
requestConfig: getGoogleVertexRequestConfig(id)
}];
}));
//#endregion
//#region src/google-vertex-image-model.ts
var GoogleVertexImageModel = class GoogleVertexImageModel {
static [WORKFLOW_SERIALIZE](model) {
return serializeModelOptions({
modelId: model.modelId,
config: model.config
});
}
static [WORKFLOW_DESERIALIZE](options) {
return new GoogleVertexImageModel(options.modelId, options.config);
}
get supportsFileInputs() {
return getGoogleVertexImageModelEntry(this.modelId).capabilities.image?.fileInput === true ? true : void 0;
}
get supportsMaskInputs() {
return this.supportsFileInputs === true ? false : void 0;
}
get provider() {
return this.config.provider;
}
constructor(modelId, config) {
this.modelId = modelId;
this.config = config;
this.specificationVersion = "v4";
this.maxImagesPerCall = 1;
}
async doGenerate(options) {
if (!getGoogleVertexRequestConfig(this.modelId).image?.gemini) throw new Error("Google image models other than Gemini are no longer supported. Use a model ID that starts with `gemini-`.");
const { prompt, size, aspectRatio, seed, providerOptions, headers, abortSignal, files, mask } = options;
const warnings = [];
if (mask != null) throw new Error("Gemini image models do not support mask-based image editing.");
if (size != null) warnings.push({
type: "unsupported",
feature: "size",
details: "This model does not support the `size` option. Use `aspectRatio` instead."
});
const userContent = [];
if (prompt != null) userContent.push({
type: "text",
text: prompt
});
if (files != null && files.length > 0) for (const file of files) if (file.type === "url") userContent.push({
type: "file",
data: {
type: "url",
url: new URL(file.url)
},
mediaType: "image/*"
});
else userContent.push({
type: "file",
data: {
type: "data",
data: typeof file.data === "string" ? file.data : new Uint8Array(file.data)
},
mediaType: file.mediaType
});
const languageModelPrompt = [{
role: "user",
content: userContent
}];
const languageModel = new GoogleLanguageModel(this.modelId, {
provider: this.config.provider,
baseURL: this.config.baseURL,
headers: this.config.headers ?? {},
fetch: this.config.fetch,
generateId: this.config.generateId ?? generateId,
supportedUrls: () => ({ "*": [/^https?:\/\/.*$/, /^gs:\/\/.*$/] })
});
const { responseModalities: _strippedResponseModalities, imageConfig: userImageConfig, ...userVertexOptions } = providerOptions?.googleVertex ?? providerOptions?.vertex ?? {};
const innerVertexOptions = {
...userVertexOptions,
responseModalities: ["IMAGE"],
imageConfig: aspectRatio != null || userImageConfig != null ? {
...userImageConfig,
...aspectRatio != null ? { aspectRatio } : {}
} : void 0
};
const result = await languageModel.doGenerate({
prompt: languageModelPrompt,
seed,
providerOptions: {
googleVertex: innerVertexOptions,
vertex: innerVertexOptions
},
headers,
abortSignal
});
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const images = [];
for (const part of result.content) if (part.type === "file" && part.mediaType.startsWith("image/") && part.data.type === "data") images.push(convertToBase64(part.data.data));
const geminiPayload = { images: images.map(() => ({})) };
return {
images,
...result.finishReason.unified === "content-filter" ? { isRetryable: false } : {},
warnings,
providerMetadata: {
googleVertex: geminiPayload,
vertex: geminiPayload
},
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: result.response?.headers
},
usage: result.usage ? {
inputTokens: result.usage.inputTokens.total,
outputTokens: result.usage.outputTokens.total,
totalTokens: (result.usage.inputTokens.total ?? 0) + (result.usage.outputTokens.total ?? 0)
} : void 0
};
}
};
//#endregion
//#region src/google-vertex-cloud-tts-speech-model.ts
const DEFAULT_VOICE = "Kore";
const DEFAULT_LANGUAGE = "en-US";
const CHIRP3_HD_VOICE_INFIX = "Chirp3-HD";
const CLOUD_TTS_SYNTHESIZE_URL = "https://texttospeech.googleapis.com/v1/text:synthesize";
/**
* Speech model for Chirp 3: HD voices on the Google Cloud Text-to-Speech API.
*
* Unlike the Gemini TTS models (which go through the Vertex
* `generateContent` endpoint via `GoogleSpeechModel`), Chirp 3: HD voices are
* served by the dedicated Cloud Text-to-Speech `text:synthesize` endpoint,
* reusing the provider's Google Cloud credentials.
*/
var GoogleVertexCloudTTSSpeechModel = class GoogleVertexCloudTTSSpeechModel {
static [WORKFLOW_SERIALIZE](model) {
return serializeModelOptions({
modelId: model.modelId,
config: model.config
});
}
static [WORKFLOW_DESERIALIZE](options) {
return new GoogleVertexCloudTTSSpeechModel(options.modelId, options.config);
}
get provider() {
return this.config.provider;
}
constructor(modelId, config) {
this.modelId = modelId;
this.config = config;
this.specificationVersion = "v4";
}
async doGenerate(options) {
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const warnings = [];
const { text, voice = DEFAULT_VOICE, outputFormat, instructions, speed, language } = options;
let voiceName;
let languageCode;
if (voice.includes(CHIRP3_HD_VOICE_INFIX)) {
voiceName = voice;
const localePrefix = voice.split(CHIRP3_HD_VOICE_INFIX)[0].replace(/-$/, "");
languageCode = language ?? (localePrefix || DEFAULT_LANGUAGE);
} else {
languageCode = language ?? DEFAULT_LANGUAGE;
voiceName = `${languageCode}-${CHIRP3_HD_VOICE_INFIX}-${voice}`;
}
if (instructions != null) warnings.push({
type: "unsupported",
feature: "instructions",
details: "Google Cloud Text-to-Speech Chirp 3: HD voices do not support the `instructions` option. It was ignored."
});
if (outputFormat != null && outputFormat !== "wav") warnings.push({
type: "unsupported",
feature: "outputFormat",
details: `Unsupported output format: ${outputFormat}. Using wav instead.`
});
const requestBody = {
input: { text },
voice: {
languageCode,
name: voiceName
},
audioConfig: {
audioEncoding: "LINEAR16",
...speed != null ? { speakingRate: speed } : {}
}
};
const { value: response, responseHeaders, rawValue: rawResponse } = await postJsonToApi({
url: CLOUD_TTS_SYNTHESIZE_URL,
headers: combineHeaders(this.config.headers ? await resolve(this.config.headers) : void 0, options.headers),
body: requestBody,
failedResponseHandler: googleVertexFailedResponseHandler,
successfulResponseHandler: createJsonResponseHandler(googleVertexCloudTTSResponseSchema),
abortSignal: options.abortSignal,
fetch: this.config.fetch
});
return {
audio: response.audioContent != null ? convertBase64ToUint8Array(response.audioContent) : /* @__PURE__ */ new Uint8Array(0),
warnings,
request: { body: JSON.stringify(requestBody) },
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders,
body: rawResponse
},
providerMetadata: { google: { mimeType: "audio/wav" } }
};
}
};
const googleVertexCloudTTSResponseSchema = lazySchema(() => zodSchema(z.object({ audioContent: z.string().nullish() })));
//#endregion
//#region src/google-vertex-tools.ts
const googleVertexTools = {
googleSearch: googleTools.googleSearch,
enterpriseWebSearch: googleTools.enterpriseWebSearch,
googleMaps: googleTools.googleMaps,
urlContext: googleTools.urlContext,
fileSearch: googleTools.fileSearch,
codeExecution: googleTools.codeExecution,
vertexRagStore: googleTools.vertexRagStore
};
//#endregion
//#region src/google-vertex-transcription-model-options.ts
const googleVertexTranscriptionProviderOptionsSchema = lazySchema(() => zodSchema(z.object({
/**
* BCP-47 language codes to recognize (e.g. `['en-US']`), or `['auto']` to let
* Chirp auto-detect the spoken language. Defaults to `['auto']`. For
* `telephony`, pass a supported explicit language code.
*/
languageCodes: z.array(z.string()).optional(),
/**
* Whether to add punctuation to the transcript. Defaults to `true`.
*/
enableAutomaticPunctuation: z.boolean().optional(),
/**
* Whether to include word-level timestamps. Defaults to `true` so the
* transcription result can include segments.
*
* Enabling word-level timestamps can reduce transcription quality and speed
* for Chirp models.
*/
enableWordTimeOffsets: z.boolean().optional(),
/**
* The Cloud Speech-to-Text region for the request (e.g. `'us'`, `'eu'`,
* `'us-central1'`). Defaults to the provider `location`. Must be a single
* DNS label (letters, digits, and hyphens).
*
* Note: Speech-to-Text regions differ from Vertex AI regions. Chirp is only
* available in specific Speech-to-Text regions and is not available in the
* `global` location.
*/
region: z.string().optional()
})));
//#endregion
//#region src/google-vertex-transcription-model.ts
function parseDurationSeconds(value) {
if (value == null) return;
const seconds = Number.parseFloat(value);
return Number.isFinite(seconds) ? seconds : void 0;
}
function convertBcp47ToIso6391(value) {
if (value == null) return;
try {
const language = new Intl.Locale(value).language;
return language.length === 2 ? language : void 0;
} catch {
return;
}
}
var GoogleVertexTranscriptionModel = class GoogleVertexTranscriptionModel {
static [WORKFLOW_SERIALIZE](model) {
return serializeModelOptions({
modelId: model.modelId,
config: model.config
});
}
static [WORKFLOW_DESERIALIZE](options) {
return new GoogleVertexTranscriptionModel(options.modelId, options.config);
}
get provider() {
return this.config.provider;
}
constructor(modelId, config) {
this.modelId = modelId;
this.config = config;
this.specificationVersion = "v4";
}
async doGenerate(options) {
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const warnings = [];
let googleOptions;
for (const provider of [
"googleVertex",
"vertex",
"google"
]) {
googleOptions = await parseProviderOptions({
provider,
providerOptions: options.providerOptions,
schema: googleVertexTranscriptionProviderOptionsSchema
});
if (googleOptions != null) break;
}
const region = googleOptions?.region ?? this.config.location;
if (!isValidHostnamePart(region)) throw new InvalidArgumentError({
argument: googleOptions?.region != null ? "region" : "location",
message: "Invalid Google Cloud Speech-to-Text region. Expected a single DNS label (letters, digits, and hyphens)."
});
const languageCodes = googleOptions?.languageCodes ?? ["auto"];
const content = typeof options.audio === "string" ? options.audio : convertUint8ArrayToBase64(options.audio);
const requestBody = {
config: {
model: this.modelId,
languageCodes,
autoDecodingConfig: {},
features: {
enableWordTimeOffsets: googleOptions?.enableWordTimeOffsets ?? true,
enableAutomaticPunctuation: googleOptions?.enableAutomaticPunctuation ?? true
}
},
content
};
const url = `https://${region === "global" ? "speech.googleapis.com" : `${region}-speech.googleapis.com`}/v2/projects/${this.config.project}/locations/${region}/recognizers/_:recognize`;
const { value: response, responseHeaders, rawValue: rawResponse } = await postJsonToApi({
url,
headers: combineHeaders(this.config.headers ? await resolve(this.config.headers) : void 0, options.headers),
body: requestBody,
failedResponseHandler: googleVertexFailedResponseHandler,
successfulResponseHandler: createJsonResponseHandler(googleVertexTranscriptionResponseSchema),
abortSignal: options.abortSignal,
fetch: this.config.fetch
});
const results = response.results ?? [];
return {
text: results.map((result) => result.alternatives?.[0]?.transcript ?? "").join(" ").trim(),
segments: results.flatMap((result) => result.alternatives?.[0]?.words?.flatMap((word) => {
const startSecond = parseDurationSeconds(word.startOffset);
const endSecond = parseDurationSeconds(word.endOffset);
return word.word == null || startSecond == null || endSecond == null ? [] : [{
text: word.word,
startSecond,
endSecond
}];
}) ?? []),
language: convertBcp47ToIso6391(results[0]?.languageCode),
durationInSeconds: parseDurationSeconds(response.metadata?.totalBilledDuration),
warnings,
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders,
body: rawResponse
}
};
}
};
const googleVertexTranscriptionResponseSchema = lazySchema(() => zodSchema(z.object({
results: z.array(z.object({
alternatives: z.array(z.object({
transcript: z.string().nullish(),
words: z.array(z.object({
word: z.string().nullish(),
startOffset: z.string().nullish(),
endOffset: z.string().nullish()
})).nullish()
})).nullish(),
languageCode: z.string().nullish()
})).nullish(),
metadata: z.object({ totalBilledDuration: z.string().nullish() }).nullish()
})));
//#endregion
//#region src/gemini-transcription/google-vertex-gemini-transcription-model-options.ts
/**
* Speech recognition options for Gemini transcription models on Vertex,
* shared by unary (`gemini-3.5-transcribe`) and live
* (`gemini-3.5-transcribe-live`) variants. Maps onto Google's
* `AudioTranscriptionConfig`.
*/
const googleVertexGeminiTranscriptionModelOptions = lazySchema(() => zodSchema(z.object({
/**
* BCP-47 language codes providing hints about the languages present in the
* audio. If omitted or empty, defaults to automatic language detection.
*/
languageCodes: z.array(z.string()).optional(),
/**
* Custom vocabulary phrases, which bias the speech recognition model
* toward recognizing specific terms.
*/
customVocabulary: z.array(z.string()).optional(),
/**
* Enables word-level timestamp generation.
*/
wordTimestamp: z.boolean().optional(),
/**
* Enables speaker diarization.
*/
diarization: z.boolean().optional(),
/**
* Transcription output formatting mode.
*
* - `VERBATIM` (default): exact literal transcript preserving filler
* words, repetitions, and false starts.
* - `SMART`: cleans up and structures the transcript in real time —
* disfluency removal, inline self-corrections, structured formatting
* (lists, numbers, dates, paragraph breaks), and grammar/casing polish.
*/
mode: z.enum(["SMART", "VERBATIM"]).optional()
})));
//#endregion
//#region src/gemini-transcription/google-vertex-gemini-transcription-model.ts
const liveWebSocketPath = "google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent";
/**
* After the input audio has ended, finish when no terminal signal
* (`turnComplete` / idle `interactionStatus`) arrives within this window.
* Trailing transcripts reset the timer.
*/
const defaultFinishGraceMs = 3e3;
/** Live transcription is only supported by `*-live` model variants. */
function isLiveTranscriptionModelId(modelId) {
return getGoogleVertexRequestConfig(modelId).transcription?.live === true;
}
/** Regional Vertex hostname (mirrors the provider's base-URL host rules). */
function vertexHost(location) {
if (location === "global") return "aiplatform.googleapis.com";
if (location === "eu" || location === "us") return `aiplatform.${location}.rep.googleapis.com`;
return `${location}-aiplatform.googleapis.com`;
}
/**
* Gemini transcription on Vertex AI. Unary variants transcribe via
* `generateContent`; live variants stream over the Vertex Live API WebSocket
* (`LlmBidiService/BidiGenerateContent`) with OAuth Bearer authentication
* from the provider's resolved headers.
*/
var GoogleVertexGeminiTranscriptionModel = class GoogleVertexGeminiTranscriptionModel {
static [WORKFLOW_SERIALIZE](model) {
return serializeModelOptions({
modelId: model.modelId,
config: model.config
});
}
static [WORKFLOW_DESERIALIZE](options) {
return new GoogleVertexGeminiTranscriptionModel(options.modelId, options.config);
}
get provider() {
return this.config.provider;
}
constructor(modelId, config) {
this.modelId = modelId;
this.config = config;
this.specificationVersion = "v4";
}
async parseOptions(providerOptions) {
for (const provider of [
"googleVertex",
"vertex",
"google"
]) {
const parsed = await parseProviderOptions({
provider,
providerOptions,
schema: googleVertexGeminiTranscriptionModelOptions
});
if (parsed != null) return parsed;
}
}
async doGenerate(options) {
if (isLiveTranscriptionModelId(this.modelId)) throw new InvalidArgumentError({
argument: "modelId",
message: `Model '${this.modelId}' only supports streaming transcription. Use experimental_streamTranscribe, or a unary model such as 'gemini-3.5-transcribe'.`
});
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const warnings = [];
const googleOptions = await this.parseOptions(options.providerOptions);
const wantsWord = options.timestampGranularities?.includes("word");
if (wantsWord && googleOptions?.wordTimestamp === false) warnings.push({
type: "unsupported",
feature: "timestampGranularities",
details: "The provider wordTimestamp option is false and overrides the requested word timings."
});
if (options.timestampGranularities?.includes("segment")) warnings.push({
type: "unsupported",
feature: "timestampGranularities",
details: "Vertex Gemini transcription returns word timings and does not provide separate caption segments."
});
const audioTranscriptionConfig = buildAudioTranscriptionConfig(googleOptions?.wordTimestamp == null && wantsWord ? {
...googleOptions,
wordTimestamp: true
} : googleOptions);
const requestBody = {
contents: [{
role: "user",
parts: [{ inlineData: {
mimeType: options.mediaType,
data: convertToBase64(options.audio)
} }]
}],
...audioTranscriptionConfig != null ? { generationConfig: { audioTranscriptionConfig } } : {}
};
const { value: response, responseHeaders, rawValue: rawResponse } = await postJsonToApi({
url: `${this.config.baseURL}/models/${this.modelId}:generateContent`,
headers: combineHeaders(this.config.headers ? await resolve(this.config.headers) : void 0, options.headers),
body: requestBody,
failedResponseHandler: googleVertexFailedResponseHandler,
successfulResponseHandler: createJsonResponseHandler(googleVertexGeminiTranscriptionResponseSchema),
abortSignal: options.abortSignal,
fetch: this.config.fetch
});
const parts = response.candidates?.[0]?.content?.parts ?? [];
const plainText = parts.map((part) => part.text ?? "").join("");
const transcriptionText = parts.map((part) => part.audioTranscription?.text ?? "").join("");
const text = plainText !== "" ? plainText : transcriptionText;
let language;
const segments = [];
const words = [];
for (const part of parts) {
const transcription = part.audioTranscription;
if (transcription == null) continue;
language ??= transcription.languageCode ?? void 0;
for (const word of transcription.words ?? []) {
const startSecond = parseOffsetSeconds(word.startOffset);
const endSecond = parseOffsetSeconds(word.endOffset);
if (word.word == null || startSecond == null || endSecond == null) continue;
const timing = {
text: word.word,
startSecond,
endSecond
};
words.push(timing);
segments.push({ ...timing });
}
}
if (wantsWord && googleOptions?.wordTimestamp !== false && words.length === 0 && text.trim() !== "") warnings.push({
type: "other",
message: "The Vertex Gemini transcription response omitted the requested word timings."
});
return {
text,
segments,
...words.length > 0 ? { words } : {},
language,
durationInSeconds: void 0,
warnings,
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders,
body: rawResponse
},
...response.usageMetadata != null ? {
usage: response.usageMetadata,
providerMetadata: { google: { usageMetadata: response.usageMetadata } }
} : {}
};
}
async doStream(options) {
if (!isLiveTranscriptionModelId(this.modelId)) throw new InvalidArgumentError({
argument: "modelId",
message: `Model '${this.modelId}' does not support streaming transcription. Use a live model such as 'gemini-3.5-transcribe-live'.`
});
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const warnings = [];
const googleOptions = await this.parseOptions(options.providerOptions);
validateLiveInputAudioFormat(options.inputAudioFormat);
const headers = combineHeaders(this.config.headers ? await resolve(this.config.headers) : void 0, options.headers);
const { project, location } = this.config;
const modelResource = `projects/${project}/locations/${location}/publishers/google/models/${this.modelId}`;
const url = new URL(`wss://${vertexHost(location)}/ws/${liveWebSocketPath}`);
const setup = {
model: modelResource,
inputAudioTranscription: buildAudioTranscriptionConfig(googleOptions) ?? {}
};
return {
request: { body: setup },
response: {
timestamp: currentDate,
modelId: this.modelId
},
stream: createVertexLiveTranscriptionStream({
webSocket: this.config.webSocket,
url,
headers,
setup,
inputAudioRate: options.inputAudioFormat.rate ?? 16e3,
finishGraceMs: this.config._internal?.finishGraceMs ?? defaultFinishGraceMs,
warnings,
audio: options.audio,
abortSignal: options.abortSignal,
includeRawChunks: options.includeRawChunks
})
};
}
};
function createVertexLiveTranscriptionStream({ webSocket, url, headers, setup, inputAudioRate, finishGraceMs, warnings, audio, abortSignal, includeRawChunks }) {
let finished = false;
let cleanup = () => {};
return new ReadableStream({
start: (controller) => {
let audioReader;
let connection;
let resolveSetupComplete;
const setupComplete = new Promise((resolvePromise) => {
resolveSetupComplete = resolvePromise;
});
let segmentCounter = 0;
let segmentBuffer = "";
let fullText = "";
let latestInterim = "";
let language;
let audioEnded = false;
let usageMetadata;
let finishTimer;
const segmentId = () => `google-segment-${segmentCounter}`;
const cancelPendingFinish = () => {
if (finishTimer != null) {
clearTimeout(finishTimer);
finishTimer = void 0;
}
};
const schedulePendingFinish = () => {
if (finished || !audioEnded) return;
cancelPendingFinish();
finishTimer = setTimeout(() => {
finishTimer = void 0;
finish();
}, finishGraceMs);
};
cleanup = (closeCode) => {
cancelPendingFinish();
if (audioReader != null) audioReader.cancel().catch(() => {});
else audio.cancel().catch(() => {});
connection?.close(closeCode);
};
const finishWithError = (error) => {
if (finished) return;
finished = true;
cleanup();
controller.error(error);
};
const completeSegment = () => {
if (segmentBuffer === "") {
if (latestInterim === "") return;
segmentBuffer = latestInterim;
}
latestInterim = "";
controller.enqueue({
type: "transcript-final",
id: segmentId(),
text: segmentBuffer
});
fullText += fullText === "" ? segmentBuffer : ` ${segmentBuffer}`;
segmentBuffer = "";
segmentCounter++;
};
const finish = () => {
if (finished) return;
completeSegment();
finished = true;
controller.enqueue({
type: "finish",
text: fullText,
segments: [],
language,
durationInSeconds: void 0,
...usageMetadata != null ? {
usage: usageMetadata,
providerMetadata: { google: { usageMetadata } }
} : {}
});
controller.close();
cleanup(1e3);
};
const sendAudio = async (socket) => {
audioReader = audio.getReader();
try {
while (true) {
const { done, value } = await audioReader.read();
if (done || finished) break;
socket.send(JSON.stringify({ realtimeInput: { audio: {
data: convertToBase64(value),
mimeType: `audio/pcm;rate=${inputAudioRate}`
} } }));
await waitForWebSocketBufferDrain(socket);
}
} finally {
audioReader.releaseLock();
audioReader = void 0;
}
if (!finished) {
socket.send(JSON.stringify({ realtimeInput: { audioStreamEnd: true } }));
audioEnded = true;
schedulePendingFinish();
}
};
connection = connectToWebSocket({
url,
headers,
webSocket,
abortSignal,
onAbort: finishWithError,
onProcessingError: finishWithError,
onOpen: (socket) => {
controller.enqueue({
type: "stream-start",
warnings
});
socket.send(JSON.stringify({ setup }));
setupComplete.then(() => finished ? void 0 : sendAudio(socket)).catch(finishWithError);
},
onMessageText: async (text) => {
if (finished) return;
const parsed = await safeParseJSON({ text });
if (!parsed.success) return;
const message = parsed.value;
if (includeRawChunks) controller.enqueue({
type: "raw",
rawValue: message
});
if (message.setupComplete != null) resolveSetupComplete();
if (message.usageMetadata != null) usageMetadata = message.usageMetadata;
if (message.error != null) {
finishWithError(new Error(message.error.message ?? "Vertex Live API error"));
return;
}
const serverContent = message.serverContent;
const interim = serverContent?.interimInputTranscription;
if (interim?.text) {
schedulePendingFinish();
latestInterim = interim.text;
controller.enqueue({
type: "transcript-partial",
id: segmentId(),
text: interim.text
});
}
const transcription = serverContent?.inputTranscription ?? message.inputTranscription;
if (transcription != null) {
if (transcription.languageCode != null) language = transcription.languageCode;
if (transcription.text) {
schedulePendingFinish();
latestInterim = "";
segmentBuffer += transcription.text;
controller.enqueue({
type: "transcript-delta",
id: segmentId(),
delta: transcription.text
});
}
if (transcription.finished === true) completeSegment();
}
if (serverContent?.turnComplete) completeSegment();
const interactionStatus = serverContent?.interactionStatus;
if (audioEnded && (interactionStatus === "IDLE" || interactionStatus === "REQUIRES_ACTION" || serverContent?.turnComplete === true && interactionStatus == null)) finish();
},
onSocketError: () => {
finishWithError(/* @__PURE__ */ new Error("Vertex Live transcription error." + (webSocket == null ? " Note: the native WebSocket implementation cannot send the Authorization header required by Vertex. Pass a header-capable WebSocket implementation (e.g. the 'ws' package) via createVertex({ webSocket })." : "")));
},
onClose: ({ code, reason }) => {
if (finished) return;
if (audioEnded) {
finish();
return;
}
finishWithError(/* @__PURE__ */ new Error(`Vertex Live transcription WebSocket closed unexpectedly before finishing (code ${code ?? "unknown"}${reason ? `, reason: ${reason}` : ""}).`));
}
});
},
cancel: () => {
if (finished) return;
finished = true;
cleanup();
}
});
}
/**
* Builds Google's `AudioTranscriptionConfig` from provider options; returns
* undefined when no options are set.
*/
function buildAudioTranscriptionConfig(options) {
if (options == null) return void 0;
const config = {};
if (options.languageCodes != null) config.languageCodes = options.languageCodes;
if (options.customVocabulary != null) config.customVocabulary = options.customVocabulary;
if (options.wordTimestamp != null) config.wordTimestamp = options.wordTimestamp;
if (options.diarization != null) config.diarization = options.diarization;
if (options.mode != null) config.mode = options.mode;
return Object.keys(config).length > 0 ? config : void 0;
}
function validateLiveInputAudioFormat(inputAudioFormat) {
if (inputAudioFormat.type !== "audio/pcm" || inputAudioFormat.rate != null && inputAudioFormat.rate !== 16e3) throw new InvalidArgumentError({
argument: "inputAudioFormat",
message: "The Gemini Live transcription API only supports 16kHz 16-bit PCM input audio."
});
}
/** Parses a Google duration offset such as `"1s"` or `"9.400s"` to seconds. */
function parseOffsetSeconds(offset) {
if (offset == null) return void 0;
const parsed = Number.parseFloat(offset);
return Number.isFinite(parsed) ? parsed : void 0;
}
const googleVertexGeminiTranscriptionWordSchema = () => z.object({
word: z.string().nullish(),
startOffset: z.string().nullish(),
endOffset: z.string().nullish()
});
const googleVertexGeminiTranscriptionResponseSchema = lazySchema(() => zodSchema(z.object({
candidates: z.array(z.object({ content: z.object({ parts: z.array(z.object({
text: z.string().nullish(),
audioTranscription: z.object({
text: z.string().nullish(),
languageCode: z.string().nullish(),
speakerLabel: z.string().nullish(),
words: z.array(googleVertexGeminiTranscriptionWordSchema()).nullish()
}).nullish()
})).nullish() }).nullish() })).nullish(),
usageMetadata: z.record(z.string(), z.unknown()).nullish()
})));
//#endregion
//#region src/google-vertex-video-model-options.ts
const googleVertexVideoModelOptionsSchema = lazySchema(() => zodSchema(z.looseObject({
pollIntervalMs: z.number().positive().nullish(),
pollTimeoutMs: z.number().positive().nullish(),
personGeneration: z.enum([
"dont_allow",
"allow_adult",
"allow_all"
]).nullish(),
negativePrompt: z.string().nullish(),
generateAudio: z.boolean().nullish(),
gcsOutputDirectory: z.string().nullish(),
referenceImages: z.array(z.object({
bytesBase64Encoded: z.string().nullish(),
gcsUri: z.string().nullish()
})).nullish()
})));
//#endregion
//#region src/google-vertex-video-model.ts
function getFirstFrameImage(options) {
return options.frameImages?.find((frame) => frame.frameType === "first_frame")?.image;
}
function resolveStartImage(options) {
return getFirstFrameImage(options) ?? options.image;
}
function getLastFrameImage(options) {
return options.frameImages?.find((frame) => frame.frameType === "last_frame")?.image;
}
function getInputReferences(options) {
if (options.frameImages != null && options.frameImages.length > 0) return;
return options.inputReferences != null && options.inputReferences.length > 0 ? options.inputReferences : void 0;
}
function convertFileToVertexImage(file, warnings) {
if (file.type === "url") {
if (file.url.startsWith("gs://")) return {
gcsUri: file.url,
mimeType: "image/png"
};
warnings.push({
type: "unsupported",
feature: "URL-based image input",
details: "Vertex AI video models require base64-encoded images or GCS URIs. URL will be ignored."
});
return;
}
return {
bytesBase64Encoded: typeof file.data === "string" ? file.data : convertUint8ArrayToBase64(file.data),
mimeType: file.mediaType || "image/png"
};
}
function convertInputReferenceImage(file, warnings) {
const image = convertFileToVertexImage(file, warnings);
return image != null ? {
image,
referenceType: "asset"
} : void 0;
}
var GoogleVertexVideoModel = class {
get provider() {
return this.config.provider;
}
get maxVideosPerCall() {
return 4;
}
constructor(modelId, config) {
this.modelId = modelId;
this.config = config;
this.specificationVersion = "v4";
}
async buildRequest(options) {
const warnings = [];
const googleVertexOptions = await parseProviderOptions({
provider: "googleVertex",
providerOptions: options.providerOptions,
schema: googleVertexVideoModelOptionsSchema
}) ?? await parseProviderOptions({
provider: "vertex",
providerOptions: options.providerOptions,
schema: googleVertexVideoModelOptionsSchema
});
const instances = [{}];
const instance = instances[0];
if (options.prompt != null) instance.prompt = options.prompt;
const startImage = resolveStartImage(options);
if (startImage != null) {
const image = convertFileToVertexImage(startImage, warnings);
if (image != null) instance.image = image;
}
const lastFrameImage = getLastFrameImage(options);
if (lastFrameImage != null) {
const lastFrame = convertFileToVertexImage(lastFrameImage, warnings);
if (lastFrame != null) instance.lastFrame = lastFrame;
}
const inputReferences = getInputReferences(options);
if (inputReferences != null) instance.referenceImages = inputReferences.flatMap((reference) => {
const converted = convertInputReferenceImage(reference, warnings);
return converted != null ? [converted] : [];
});
else if (googleVertexOptions?.referenceImages != null) instance.referenceImages = googleVertexOptions.referenceImages;
const parameters = { sampleCount: options.n };
if (options.aspectRatio) parameters.aspectRatio = options.aspectRatio;
if (options.resolution) parameters.resolution = {
"1280x720": "720p",
"1920x1080": "1080p",
"3840x2160": "4k"
}[options.resolution] || options.resolution;
if (options.duration) parameters.durationSeconds = options.duration;
if (options.seed != null) parameters.seed = options.seed;
const generateAudio = options.generateAudio ?? googleVertexOptions?.generateAudio;
if (generateAudio != null) parameters.generateAudio = generateAudio;
if (googleVertexOptions != null) {
const opts = googleVertexOptions;
if (opts.personGeneration !== void 0 && opts.personGeneration !== null) parameters.personGeneration = opts.personGeneration;
if (opts.negativePrompt !== void 0 && opts.negativePrompt !== null) parameters.negativePrompt = opts.negativePrompt;
if (opts.gcsOutputDirectory !== void 0 && opts.gcsOutputDirectory !== null) parameters.gcsOutputDirectory = opts.gcsOutputDirectory;
for (const [key, value] of Object.entries(opts)) if (![
"pollIntervalMs",
"pollTimeoutMs",
"personGeneration",
"negativePrompt",
"generateAudio",
"gcsOutputDirectory",
"referenceImages"
].includes(key)) parameters[key] = value;
}
return {
instances,
parameters,
warnings,
googleVertexOptions
};
}
buildCompletedResult({ finalOperation, responseHeaders, warnings, currentDate }) {
const response = finalOperation.response;
if (!response?.videos || response.videos.length === 0) throw new AISDKError({
name: "VERTEX_VIDEO_GENERATION_ERROR",
message: `No videos in response. Response: ${JSON.stringify(finalOperation)}`
});
const videos = [];
const videoMetadata = [];
for (const video of response.videos) if (video.bytesBase64Encoded) {
videos.push({
type: "base64",
data: video.bytesBase64Encoded,
mediaType: video.mimeType || "video/mp4"
});
videoMetadata.push({ mimeType: video.mimeType });
} else if (video.gcsUri) {
videos.push({
type: "url",
url: video.gcsUri,
mediaType: video.mimeType || "video/mp4"
});
videoMetadata.push({
gcsUri: video.gcsUri,
mimeType: video.mimeType
});
}
if (videos.length === 0) throw new AISDKError({
name: "VERTEX_VIDEO_GENERATION_ERROR",
message: "No valid videos in response"
});
return {
status: "completed",
videos,
warnings,
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders
},
providerMetadata: (() => {
const payload = { videos: videoMetadata };
return {
googleVertex: payload,
"google-vertex": payload,
vertex: payload
};
})()
};
}
async doStart(options) {
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const { instances, parameters, warnings } = await this.buildRequest(options);
const { value: operation, responseHeaders } = await postJsonToApi({
url: `${this.config.baseURL}/models/${this.modelId}:predictLongRunning`,
headers: combineHeaders(await resolve(this.config.headers), options.headers),
body: {
instances,
parameters
},
successfulResponseHandler: createJsonResponseHandler(googleVertexOperationSchema),
failedResponseHandler: googleVertexFailedResponseHandler,
abortSignal: options.abortSignal,
fetch: this.config.fetch
});
const operationName = operation.name;
if (!operationName) throw new AISDKError({
name: "VERTEX_VIDEO_GENERATION_ERROR",
message: "No operation name returned from API"
});
return {
operation: { operationName },
warnings,
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders
}
};
}
async doStatus(options) {
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
const { operationName } = options.operation;
const { value: statusOperation, responseHeaders } = await postJsonToApi({
url: `${this.config.baseURL}/models/${this.modelId}:fetchPredictOperation`,
headers: combineHeaders(await resolve(this.config.headers), options.headers),
body: { operationName },
successfulResponseHandler: createJsonResponseHandler(googleVertexOperationSchema),
failedResponseHandler: googleVertexFailedResponseHandler,
abortSignal: options.abortSignal,
fetch: this.config.fetch
});
if (!statusOperation.done) return {
status: "pending",
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders
}
};
if (statusOperation.error) return {
status: "error",
error: `Video generation failed: ${statusOperation.error.message}`,
response: {
timestamp: currentDate,
modelId: this.modelId,
headers: responseHeaders
}
};
return this.buildCompletedResult({
finalOperation: statusOperation,
responseHeaders,
warnings: [],
currentDate
});
}
};
const googleVertexOperationSchema = lazySchema(() => zodSchema(z.object({
name: z.string().nullish(),
done: z.boolean().nullish(),
error: z.object({
code: z.number().nullish(),
message: z.string(),
status: z.string().nullish()
}).nullish(),
response: z.object({
videos: z.array(z.object({
bytesBase64Encoded: z.string().nullish(),
gcsUri: z.string().nullish(),
mimeType: z.string().nullish()
})).nullish(),
raiMediaFilteredCount: z.number().nullish()
}).nullish()
})));
//#endregion
//#region src/google-vertex-provider-base.ts
const EXPRESS_MODE_BASE_URL = "https://aiplatform.googleapis.com/v1/publishers/google";
function isEndpointModelId(modelId) {
return getGoogleVertexRequestConfig(modelId).language?.endpoint === true;
}
function createExpressModeFetch(apiKey, customFetch) {
return async (url, init) => {
const resolvedApiKey = await resolve(apiKey);
if (typeof apiKey !== "string" && (typeof resolvedApiKey !== "string" || !resolvedApiKey.trim())) throw new LoadAPIKeyError({ message: "Google Vertex API key must be a non-empty string." });
const modifiedInit = {
...init,
headers: {
...init?.headers ? normalizeHeaders(init.headers) : {},
"x-goog-api-key": resolvedApiKey
}
};
return (customFetch ?? fetch)(url.toString(), modifiedInit);
};
}
/**
* Create a Google Vertex AI provider instance.
*/
function createGoogleVertex$1(options = {}) {
const apiKey = typeof options.apiKey === "string" || options.apiKey == null ? loadOptionalSetting({
settingValue: options.apiKey,
environmentVariableName: "GOOGLE_VERTEX_API_KEY"
}) : options.apiKey;
const loadGoogleVertexProject = () => loadSetting({
settingValue: options.project,
settingName: "project",
environmentVariableName: "GOOGLE_VERTEX_PROJECT",
description: "Google Vertex project"
});
const loadGoogleVertexLocation = () => {
const location = loadSetting({
settingValue: options.location,
settingName: "location",
environmentVariableName: "GOOGLE_VERTEX_LOCATION",
description: "Google Vertex location"
});
if (!isValidHostnamePart(location)) throw new InvalidArgumentError({
argument: "location",
message: "Invalid Google Vertex location. Expected a single DNS label (letters, digits, and hyphens). Use `baseURL` for custom endpoints."
});
return location;
};
const loadBaseURL = ({ endpoint = false } = {}) => {
if (apiKey) return withoutTrailingSlash(options.baseURL) ?? EXPRESS_MODE_BASE_URL;
const baseURL = withoutTrailingSlash(options.baseURL);
if (baseURL != null) return baseURL;
const region = loadGoogleVertexLocation();
const project = loadGoogleVertexProject();
const getHost = () => {
if (region === "global") return "aiplatform.googleapis.com";
else if (region === "eu" || region === "us") return `aiplatform.${region}.rep.googleapis.com`;
else return `${region}-aiplatform.googleapis.com`;
};
return `https://${getHost()}/v1beta1/projects/${project}/locations/${region}${endpoint ? "" : "/publishers/google"}`;
};
const createConfig = (name, { endpoint = false } = {}) => {
const getHeaders = async () => {
const originalHeaders = await resolve(options.headers ?? {});
return withUserAgentSuffix(originalHeaders, `ai-sdk-google-vertex/${VERSION}`);
};
return {
provider: `google.vertex.${name}`,
headers: getHeaders,
fetch: apiKey ? createExpressModeFetch(apiKey, options.fetch) : options.fetch,
baseURL: loadBaseURL({ endpoint })
};
};
const createChatModel = (modelId) => {
const endpoint = isEndpointModelId(modelId);
if (endpoint && apiKey) throw new Error("Google Vertex tuned models do not support Express Mode API keys. Use standard Google Cloud credentials instead.");
return new GoogleLanguageModel(modelId, {
...createConfig("chat", { endpoint }),
generateId: options.generateId ?? generateId,
supportedUrls: () => ({ "*": [/^https?:\/\/.*$/, /^gs:\/\/.*$/] }),
downloadToolResultFiles: {
maxBytes: options.toolResultDownloads?.maxBytes ?? 7340032,
supportsGoogleCloudStorageUrls: true,
fetch: options.fetch
}
});
};
const createInteractionsModel = (modelIdOrAgent) => {
if (apiKey) throw new Error("Google Vertex Interactions models do not support Express Mode API keys. Use standard Google Cloud credentials instead.");
return new GoogleInteractionsLanguageModel(modelIdOrAgent, {
...createConfig("interactions", { endpoint: true }),
generateId: options.generateId ?? generateId
});
};
const createEmbeddingModel = (modelId) => new GoogleVertexEmbeddingModel(modelId, createConfig("embedding"));
const createImageModel = (modelId) => new GoogleVertexImageModel(modelId, {
...createConfig("image"),
generateId: options.generateId ?? generateId
});
const createVideoModel = (modelId) => new GoogleVertexVideoModel(modelId, {
...createConfig("video"),
generateId: options.generateId ?? generateId
});
const createSpeechModel = (modelId) => {
if (getGoogleVertexRequestConfig(modelId).speech?.api === "cloud-tts") {
if (apiKey) throw new Error("Google Vertex Chirp speech models do not support Express Mode API keys. Use standard Google Cloud credentials instead.");
const config = createConfig("speech");
return new GoogleVertexCloudTTSSpeechModel(modelId, {
provider: config.provider,
headers: config.headers,
fetch: config.fetch
});
}
return new GoogleSpeechModel(modelId, createConfig("speech"));
};
const createTranscriptionModel = (modelId) => {
if (apiKey) throw new Error("Google Vertex transcription models do not support Express Mode API keys. Use standard Google Cloud credentials instead.");
const config = createConfig("transcription");
if (getGoogleVertexRequestConfig(modelId).transcription?.api === "gemini") return new GoogleVertexGeminiTranscriptionModel(modelId, {
provider: config.provider,
baseURL: loadBaseURL(),
headers: config.headers,
fetch: config.fetch,
webSocket: options.webSocket,
project: loadGoogleVertexProject(),
location: loadGoogleVertexLocation()
});
return new GoogleVertexTranscriptionModel(modelId, {
provider: config.provider,
headers: config.headers,
fetch: config.fetch,
project: loadGoogleVertexProject(),
location: loadGoogleVertexLocation()
});
};
const provider = function(modelId) {
if (new.target) throw new Error("The Google Vertex AI model function cannot be called with the new keyword.");
return createChatModel(modelId);
};
provider.specificationVersion = "v4";
provider.languageModel = createChatModel;
provider.interactions = createInteractionsModel;
provider.embeddingModel = createEmbeddingModel;
provider.textEmbeddingModel = createEmbeddingModel;
provider.image = createImageModel;
provider.imageModel = createImageModel;
provider.video = createVideoModel;
provider.videoModel = createVideoModel;
provider.speech = createSpeechModel;
provider.speechModel = createSpeechModel;
provider.transcription = createTranscriptionModel;
provider.transcriptionModel = createTranscriptionModel;
provider.tools = googleVertexTools;
return provider;
}
//#endregion
//#region src/edge/google-vertex-auth-edge.ts
const loadCredentials = async () => {
try {
return {
clientEmail: loadSetting({
settingValue: void 0,
settingName: "clientEmail",
environmentVariableName: "GOOGLE_CLIENT_EMAIL",
description: "Google client email"
}),
privateKey: loadSetting({
settingValue: void 0,
settingName: "privateKey",
environmentVariableName: "GOOGLE_PRIVATE_KEY",
description: "Google private key"
}),
privateKeyId: loadOptionalSetting({
settingValue: void 0,
environmentVariableName: "GOOGLE_PRIVATE_KEY_ID"
})
};
} catch (error) {
throw new Error(`Failed to load Google credentials: ${error.message}`);
}
};
const base64url = (str) => {
return btoa(str).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
};
const importPrivateKey = async (pemKey) => {
const pemContents = pemKey.replace("-----BEGIN PRIVATE KEY-----", "").replace("-----END PRIVATE KEY-----", "").replace(/\s/g, "");
const binaryString = atob(pemContents);
const binaryData = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) binaryData[i] = binaryString.charCodeAt(i);
return await crypto.subtle.importKey("pkcs8", binaryData, {
name: "RSASSA-PKCS1-v1_5",
hash: "SHA-256"
}, true, ["sign"]);
};
const buildJwt = async (credentials) => {
const now = Math.floor(Date.now() / 1e3);
const header = {
alg: "RS256",
typ: "JWT"
};
if (credentials.privateKeyId) header.kid = credentials.privateKeyId;
const payload = {
iss: credentials.clientEmail,
scope: "https://www.googleapis.com/auth/cloud-platform",
aud: "https://oauth2.googleapis.com/token",
exp: now + 3600,
iat: now
};
const privateKey = await importPrivateKey(credentials.privateKey);
const signingInput = `${base64url(JSON.stringify(header))}.${base64url(JSON.stringify(payload))}`;
const data = new TextEncoder().encode(signingInput);
const signature = await crypto.subtle.sign("RSASSA-PKCS1-v1_5", privateKey, data);
const signatureBase64 = base64url(String.fromCharCode(...new Uint8Array(signature)));
return `${base64url(JSON.stringify(header))}.${base64url(JSON.stringify(payload))}.${signatureBase64}`;
};
/**
* Generate an authentication token for Google Vertex AI in a manner compatible
* with the Edge runtime.
*/
async function generateAuthToken(credentials) {
const creds = credentials || await loadCredentials();
const jwt = await buildJwt(creds);
const response = await fetch("https://oauth2.googleapis.com/token", {
method: "POST",
headers: withUserAgentSuffix({ "Content-Type": "application/x-www-form-urlencoded" }, `ai-sdk-google-vertex/${VERSION}`, getRuntimeEnvironmentUserAgent()),
body: new URLSearchParams({
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
assertion: jwt
})
});
if (!response.ok) throw new Error(`Token request failed: ${response.statusText}`);
return (await response.json()).access_token;
}
//#endregion
//#region src/edge/google-vertex-provider-edge.ts
function createGoogleVertex(options = {}) {
if (typeof options.apiKey === "string" || options.apiKey == null ? loadOptionalSetting({
settingValue: options.apiKey,
environmentVariableName: "GOOGLE_VERTEX_API_KEY"
}) : options.apiKey) return createGoogleVertex$1(options);
return createGoogleVertex$1({
...options,
headers: async () => ({
Authorization: `Bearer ${await generateAuthToken(options.googleCredentials)}`,
...await resolve(options.headers)
})
});
}
/**
* Default Google Vertex AI provider instance.
*/
const googleVertex = createGoogleVertex();
//#endregion
export { createGoogleVertex, createGoogleVertex as createVertex, googleVertex, googleVertex as vertex };
//# sourceMappingURL=index.js.map