@just-every/ensemble
Version:
LLM provider abstraction layer with unified streaming interface
1,134 lines • 106 kB
JavaScript
import { GoogleGenAI, Type, FunctionCallingConfigMode, Modality, MediaResolution, } from '@google/genai';
import { v4 as uuidv4 } from 'uuid';
import { BaseModelProvider } from './base_provider.js';
import { costTracker } from '../utils/cost_tracker.js';
import { log_llm_error, log_llm_request, log_llm_response } from '../utils/llm_logger.js';
import { isPaused } from '../utils/pause_controller.js';
import { appendMessageWithImage, normalizeImageDataUrl, resizeAndTruncateForGemini, resizeDataUrl, } from '../utils/image_utils.js';
import { hasEventHandler } from '../utils/event_controller.js';
import { truncateLargeValues } from '../utils/truncate_utils.js';
function convertParameterToGeminiFormat(param) {
let type = Type.STRING;
switch (param.type) {
case 'string':
type = Type.STRING;
break;
case 'number':
type = Type.NUMBER;
break;
case 'boolean':
type = Type.BOOLEAN;
break;
case 'object':
type = Type.OBJECT;
break;
case 'array':
type = Type.ARRAY;
break;
case 'null':
type = Type.STRING;
console.warn("Mapping 'null' type to STRING");
break;
default:
console.warn(`Unsupported parameter type '${param.type}'. Defaulting to STRING.`);
type = Type.STRING;
}
const result = { type, description: param.description };
if (type === Type.ARRAY) {
if (param.items) {
let itemType;
let itemEnum;
let itemProperties;
if (typeof param.items === 'object') {
itemType = param.items.type;
itemEnum = param.items.enum;
if ('properties' in param.items) {
itemProperties = param.items.properties;
}
}
if (itemType === 'object' || itemProperties) {
result.items = { type: Type.STRING };
result.description = `${result.description || 'Array parameter'} (Each item should be a JSON-encoded object)`;
if (itemProperties) {
const propNames = Object.keys(itemProperties);
result.description += `. Expected properties: ${propNames.join(', ')}`;
}
}
else if (itemType) {
result.items = {
type: itemType === 'string'
? Type.STRING
: itemType === 'number'
? Type.NUMBER
: itemType === 'boolean'
? Type.BOOLEAN
: itemType === 'null'
? Type.STRING
: Type.STRING,
};
if (itemEnum) {
if (typeof itemEnum === 'function') {
console.warn('Gemini provider does not support async enum functions in array items');
}
else {
result.items.enum = itemEnum;
}
}
}
else {
result.items = { type: Type.STRING };
}
}
else {
result.items = { type: Type.STRING };
}
}
else if (type === Type.OBJECT) {
if (param.properties && typeof param.properties === 'object') {
result.properties = {};
for (const [propName, propSchema] of Object.entries(param.properties)) {
result.properties[propName] = convertParameterToGeminiFormat(propSchema);
}
}
else {
result.properties = {};
}
}
else if (param.enum) {
if (typeof param.enum === 'function') {
console.warn('Gemini provider does not support async enum functions. Enum will be omitted.');
}
else {
result.format = 'enum';
result.enum = param.enum;
}
}
return result;
}
async function resolveAsyncEnums(params) {
if (!params || typeof params !== 'object') {
return params;
}
const resolved = { ...params };
if (resolved.properties) {
const resolvedProps = {};
for (const [key, value] of Object.entries(resolved.properties)) {
if (value && typeof value === 'object') {
const propCopy = { ...value };
if (typeof propCopy.enum === 'function') {
try {
const enumValue = await propCopy.enum();
if (Array.isArray(enumValue) && enumValue.length > 0) {
propCopy.enum = enumValue;
}
else {
delete propCopy.enum;
}
}
catch {
delete propCopy.enum;
}
}
resolvedProps[key] = await resolveAsyncEnums(propCopy);
}
else {
resolvedProps[key] = value;
}
}
resolved.properties = resolvedProps;
}
return resolved;
}
async function convertToGeminiFunctionDeclarations(tools) {
const declarations = await Promise.all(tools.map(async (tool) => {
if (tool.definition.function.name === 'google_web_search' || tool.definition.function.name === 'code_execution') {
return null;
}
const resolvedParams = await resolveAsyncEnums(tool.definition?.function?.parameters);
const toolParams = resolvedParams?.properties;
const properties = {};
if (toolParams) {
for (const [name, param] of Object.entries(toolParams)) {
properties[name] = convertParameterToGeminiFormat(param);
}
}
else {
console.warn(`Tool ${tool.definition?.function?.name || 'Unnamed Tool'} has missing or invalid parameters definition.`);
}
return {
name: tool.definition.function.name,
description: tool.definition.function.description,
parameters: {
type: Type.OBJECT,
properties,
required: Array.isArray(resolvedParams?.required) ? resolvedParams.required : [],
},
};
}));
return declarations.filter(Boolean);
}
export function getImageMimeType(imageData) {
if (imageData.includes('data:image/png'))
return 'image/png';
if (imageData.includes('data:image/jpeg'))
return 'image/jpeg';
if (imageData.includes('data:image/gif'))
return 'image/gif';
if (imageData.includes('data:image/webp'))
return 'image/webp';
return 'image/png';
}
function inferImageMimeTypeFromUrl(src) {
try {
const url = new URL(src);
const path = url.pathname.toLowerCase();
if (path.endsWith('.png'))
return 'image/png';
if (path.endsWith('.jpg') || path.endsWith('.jpeg'))
return 'image/jpeg';
if (path.endsWith('.webp'))
return 'image/webp';
if (path.endsWith('.gif'))
return 'image/gif';
if (path.endsWith('.bmp'))
return 'image/bmp';
if (path.endsWith('.tif') || path.endsWith('.tiff'))
return 'image/tiff';
if (path.endsWith('.svg'))
return 'image/svg+xml';
}
catch {
}
const lower = src.toLowerCase();
if (lower.includes('.png'))
return 'image/png';
if (lower.includes('.jpg') || lower.includes('.jpeg'))
return 'image/jpeg';
if (lower.includes('.webp'))
return 'image/webp';
if (lower.includes('.gif'))
return 'image/gif';
if (lower.includes('.bmp'))
return 'image/bmp';
if (lower.includes('.tif') || lower.includes('.tiff'))
return 'image/tiff';
if (lower.includes('.svg'))
return 'image/svg+xml';
return 'image/jpeg';
}
export function cleanBase64Data(imageData) {
return imageData.replace(/^data:image\/[a-z]+;base64,/, '');
}
function formatGroundingChunks(chunks) {
return chunks
.filter(c => c?.web?.uri)
.map((c, i) => `${i + 1}. ${c.web.title || 'Untitled'} – ${c.web.uri}`)
.join('\n');
}
function normalizeGroundingChunk(chunk) {
if (!chunk || typeof chunk !== 'object')
return null;
const webUri = chunk?.web?.uri;
const webTitle = chunk?.web?.title;
const imageUri = chunk?.image?.imageUri || chunk?.image?.image_uri || chunk?.image_uri;
const imageLandingUri = chunk?.image?.uri || chunk?.uri;
const uri = webUri || imageLandingUri;
if (!uri && !imageUri)
return null;
return {
...(uri ? { uri } : {}),
...(imageUri ? { image_uri: imageUri } : {}),
...(webTitle ? { title: webTitle } : {}),
};
}
function dedupeGroundingChunks(chunks) {
const seen = new Set();
const out = [];
for (const chunk of chunks) {
const key = `${chunk.uri || ''}|${chunk.image_uri || ''}|${chunk.title || ''}`;
if (seen.has(key))
continue;
seen.add(key);
out.push(chunk);
}
return out;
}
function mergeImageMetadata(target, source) {
const next = {
...target,
model: source.model || target.model,
};
if (source.grounding) {
const t = target.grounding || {};
const s = source.grounding;
next.grounding = {
...t,
...s,
imageSearchQueries: Array.from(new Set([...(t.imageSearchQueries || []), ...(s.imageSearchQueries || [])])),
webSearchQueries: Array.from(new Set([...(t.webSearchQueries || []), ...(s.webSearchQueries || [])])),
groundingChunks: dedupeGroundingChunks([...(t.groundingChunks || []), ...(s.groundingChunks || [])]),
groundingSupports: [...(t.groundingSupports || []), ...(s.groundingSupports || [])],
};
}
next.thought_signatures = Array.from(new Set([...(target.thought_signatures || []), ...(source.thought_signatures || [])]));
next.thoughts = [...(target.thoughts || []), ...(source.thoughts || [])];
next.citations = dedupeGroundingChunks([...(target.citations || []), ...(source.citations || [])]);
return next;
}
async function addImagesToInput(input, images, source) {
for (const [image_id, imageData] of Object.entries(images)) {
const processedImageData = await resizeAndTruncateForGemini(imageData);
const mimeType = getImageMimeType(processedImageData);
const cleanedImageData = cleanBase64Data(processedImageData);
input.push({
role: 'user',
parts: [
{
text: `[image #${image_id}] from the ${source}`,
},
{
inlineData: {
mimeType: mimeType,
data: cleanedImageData,
},
},
],
});
}
return input;
}
function normalizeThoughtSignature(value) {
if (typeof value !== 'string') {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
function extractThoughtSignatureFromMessage(msg) {
if (!msg || typeof msg !== 'object') {
return null;
}
const direct = normalizeThoughtSignature(msg.thought_signature);
if (direct) {
return direct;
}
const candidate = msg;
if (candidate.type !== 'thinking') {
return null;
}
if (typeof candidate.signature === 'string') {
return normalizeThoughtSignature(candidate.signature);
}
if (!Array.isArray(candidate.signature)) {
return null;
}
for (const part of candidate.signature) {
if (typeof part === 'string') {
const parsed = normalizeThoughtSignature(part);
if (parsed) {
return parsed;
}
continue;
}
if (part && typeof part === 'object' && 'text' in part) {
const parsed = normalizeThoughtSignature(part.text);
if (parsed) {
return parsed;
}
}
}
return null;
}
async function convertToGeminiContents(model, messages) {
let contents = [];
let pendingFunctionCallParts = [];
const flushPendingFunctionCalls = () => {
if (pendingFunctionCallParts.length === 0) {
return;
}
contents.push({
role: 'model',
parts: pendingFunctionCallParts,
});
pendingFunctionCallParts = [];
};
for (const msg of messages) {
if (msg.type === 'function_call') {
let args = {};
try {
const parsedArgs = JSON.parse(msg.arguments || '{}');
args = typeof parsedArgs === 'object' && parsedArgs !== null ? parsedArgs : { value: parsedArgs };
}
catch (e) {
console.error(`Failed to parse function call arguments for ${msg.name}:`, truncateLargeValues(msg.arguments), e);
args = {
error: 'Invalid JSON arguments provided',
raw_args: msg.arguments,
};
}
const thoughtSignature = extractThoughtSignatureFromMessage(msg);
pendingFunctionCallParts.push({
functionCall: {
name: msg.name,
args,
},
...(thoughtSignature ? { thoughtSignature } : {}),
});
}
else if (msg.type === 'function_call_output') {
flushPendingFunctionCalls();
let textOutput = '';
if (typeof msg.output === 'string') {
textOutput = msg.output;
}
else {
textOutput = JSON.stringify(msg.output);
}
const message = {
role: 'user',
parts: [
{
functionResponse: {
name: msg.name,
response: { content: textOutput || '' },
},
},
],
};
contents = await appendMessageWithImage(model, contents, message, {
read: () => textOutput,
write: value => {
message.parts[0].functionResponse.response.content = value;
return message;
},
}, addImagesToInput);
}
else {
flushPendingFunctionCalls();
const role = msg.role === 'assistant' ? 'model' : 'user';
const thoughtSignature = msg.type === 'thinking' ? extractThoughtSignatureFromMessage(msg) : null;
if (Array.isArray(msg.content)) {
const parts = [];
for (const item of msg.content) {
if (item.type === 'input_text') {
parts.push({
thought: msg.type === 'thinking',
text: item.text || '',
});
}
else if (item.type === 'input_image' || item.type === 'image') {
const normalized = normalizeImageDataUrl({
data: 'data' in item ? item.data : undefined,
image_url: 'image_url' in item ? item.image_url : undefined,
url: 'url' in item ? item.url : undefined,
mime_type: 'mime_type' in item ? item.mime_type : undefined,
});
const imageUrl = normalized.dataUrl || normalized.url || ('image_url' in item ? item.image_url : '');
if (imageUrl.startsWith('data:')) {
const match = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
if (match) {
const mimeType = match[1];
const base64Data = match[2];
const processedData = await resizeAndTruncateForGemini(imageUrl);
const processedMatch = processedData.match(/^data:([^;]+);base64,(.+)$/);
if (processedMatch) {
parts.push({
inlineData: {
mimeType: processedMatch[1],
data: processedMatch[2],
},
});
}
else {
parts.push({
inlineData: {
mimeType: mimeType,
data: base64Data,
},
});
}
}
}
else if (imageUrl.startsWith('http://') || imageUrl.startsWith('https://')) {
parts.push({
fileData: {
mimeType: inferImageMimeTypeFromUrl(imageUrl),
fileUri: imageUrl,
},
});
}
}
}
if (thoughtSignature && parts.length > 0) {
parts[parts.length - 1] = {
...parts[parts.length - 1],
thoughtSignature,
};
}
if (parts.length > 0) {
const message = { role, parts };
contents.push(message);
}
}
else {
let textContent = '';
if (typeof msg.content === 'string') {
textContent = msg.content;
}
else {
textContent = JSON.stringify(msg.content);
}
const message = {
role,
parts: [
{
thought: msg.type === 'thinking',
text: textContent.trim(),
...(thoughtSignature ? { thoughtSignature } : {}),
},
],
};
contents = await appendMessageWithImage(model, contents, message, {
read: () => textContent,
write: value => {
message.parts[0].text = value;
return message;
},
}, addImagesToInput);
}
}
}
flushPendingFunctionCalls();
return contents;
}
const THINKING_BUDGET_CONFIGS = {
'-low': 0,
'-medium': 2048,
'-high': 12288,
'-max': 24576,
};
const GEMINI_31_FLASH_IMAGE_05K_DIMENSIONS = {
'1:1': { width: 512, height: 512 },
'1:4': { width: 256, height: 1024 },
'1:8': { width: 192, height: 1536 },
'2:3': { width: 424, height: 632 },
'3:2': { width: 632, height: 424 },
'3:4': { width: 448, height: 600 },
'4:1': { width: 1024, height: 256 },
'4:3': { width: 600, height: 448 },
'4:5': { width: 464, height: 576 },
'5:4': { width: 576, height: 464 },
'8:1': { width: 1536, height: 192 },
'9:16': { width: 384, height: 688 },
'16:9': { width: 688, height: 384 },
'21:9': { width: 792, height: 168 },
};
const GEMINI_3_PRO_IMAGE_DIMENSION_PRESETS = {
'1024x1024': { ar: '1:1', imageSize: '1K' },
'848x1264': { ar: '2:3', imageSize: '1K' },
'1264x848': { ar: '3:2', imageSize: '1K' },
'896x1200': { ar: '3:4', imageSize: '1K' },
'1200x896': { ar: '4:3', imageSize: '1K' },
'928x1152': { ar: '4:5', imageSize: '1K' },
'1152x928': { ar: '5:4', imageSize: '1K' },
'768x1376': { ar: '9:16', imageSize: '1K' },
'1376x768': { ar: '16:9', imageSize: '1K' },
'1584x672': { ar: '21:9', imageSize: '1K' },
'2048x2048': { ar: '1:1', imageSize: '2K' },
'1696x2528': { ar: '2:3', imageSize: '2K' },
'2528x1696': { ar: '3:2', imageSize: '2K' },
'1792x2400': { ar: '3:4', imageSize: '2K' },
'2400x1792': { ar: '4:3', imageSize: '2K' },
'1856x2304': { ar: '4:5', imageSize: '2K' },
'2304x1856': { ar: '5:4', imageSize: '2K' },
'1536x2752': { ar: '9:16', imageSize: '2K' },
'2752x1536': { ar: '16:9', imageSize: '2K' },
'3168x1344': { ar: '21:9', imageSize: '2K' },
'4096x4096': { ar: '1:1', imageSize: '4K' },
'3392x5056': { ar: '2:3', imageSize: '4K' },
'5056x3392': { ar: '3:2', imageSize: '4K' },
'3584x4800': { ar: '3:4', imageSize: '4K' },
'4800x3584': { ar: '4:3', imageSize: '4K' },
'3712x4608': { ar: '4:5', imageSize: '4K' },
'4608x3712': { ar: '5:4', imageSize: '4K' },
'3072x5504': { ar: '9:16', imageSize: '4K' },
'5504x3072': { ar: '16:9', imageSize: '4K' },
'6336x2688': { ar: '21:9', imageSize: '4K' },
};
export class GeminiProvider extends BaseModelProvider {
_client;
apiKey;
constructor(apiKey) {
super('google');
this.apiKey = apiKey;
}
get client() {
if (!this._client) {
const apiKey = this.apiKey || process.env.GOOGLE_API_KEY;
if (!apiKey) {
throw new Error('Failed to initialize Gemini client. GOOGLE_API_KEY is missing or not provided.');
}
this._client = new GoogleGenAI({
apiKey: apiKey,
vertexai: false,
httpOptions: { apiVersion: 'v1beta' },
});
}
return this._client;
}
async createEmbedding(input, model, agent, opts) {
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
let finalRequestId = requestId;
try {
let actualModelId = model.startsWith('gemini/') ? model.substring(7) : model;
let thinkingConfig = null;
for (const [suffix, budget] of Object.entries(THINKING_BUDGET_CONFIGS)) {
if (actualModelId.endsWith(suffix)) {
thinkingConfig = { thinkingBudget: budget };
actualModelId = actualModelId.slice(0, -suffix.length);
break;
}
}
console.log(`[Gemini] Generating embedding with model ${actualModelId}${opts?.dimensions ? ` (dimensions: ${opts.dimensions})` : ''}`);
const payload = {
model: actualModelId,
contents: input,
config: {
taskType: opts?.taskType ?? 'SEMANTIC_SIMILARITY',
...(opts?.dimensions && { outputDimensionality: opts.dimensions }),
},
};
if (thinkingConfig) {
payload.config.thinkingConfig = thinkingConfig;
}
const loggedRequestId = log_llm_request(agent.agent_id || 'default', 'gemini', actualModelId, {
...payload,
input_length: Array.isArray(input) ? input.length : 1,
}, new Date(), requestId, agent.tags);
finalRequestId = loggedRequestId;
const response = await this.client.models.embedContent(payload);
console.log('[Gemini] Embedding response structure:', JSON.stringify(response, (key, value) => key === 'values' && Array.isArray(value) && value.length > 10
? `[${value.length} items]`
: value, 2));
if (!response.embeddings || !Array.isArray(response.embeddings)) {
console.error('[Gemini] Unexpected embedding response structure:', truncateLargeValues(response));
throw new Error('Invalid embedding response structure from Gemini API');
}
const estimatedTokens = typeof input === 'string'
? Math.ceil(input.length / 4)
: input.reduce((sum, text) => sum + Math.ceil(text.length / 4), 0);
let extractedValues = [];
let dimensions = 0;
if (response.embeddings.length > 0) {
if (response.embeddings[0].values) {
extractedValues = response.embeddings.map(e => e.values);
dimensions = extractedValues[0].length;
}
else {
console.warn('[Gemini] Could not find expected "values" property in embeddings response');
extractedValues = response.embeddings;
dimensions = Array.isArray(extractedValues[0]) ? extractedValues[0].length : 0;
}
}
costTracker.addUsage({
model: actualModelId,
input_tokens: estimatedTokens,
output_tokens: 0,
metadata: {
dimensions,
},
});
log_llm_response(finalRequestId, {
model: actualModelId,
dimensions,
vector_count: extractedValues.length,
estimated_tokens: estimatedTokens,
});
if (Array.isArray(input) && input.length > 1) {
return extractedValues;
}
else {
let result;
if (Array.isArray(extractedValues) && extractedValues.length >= 1) {
const firstValue = extractedValues[0];
if (Array.isArray(firstValue)) {
result = firstValue;
}
else {
console.error('[Gemini] Unexpected format in embedding result:', truncateLargeValues(firstValue));
result = [];
}
}
else {
result = [];
}
return result;
}
}
catch (error) {
log_llm_error(finalRequestId, error);
console.error('[Gemini] Error generating embedding:', truncateLargeValues(error));
throw error;
}
}
async *retryStreamOnIncompleteJson(requestFn, maxRetries = 2) {
let attempts = 0;
while (attempts <= maxRetries) {
try {
const stream = await requestFn();
for await (const chunk of stream) {
yield chunk;
}
return;
}
catch (error) {
attempts++;
const errorMsg = error instanceof Error ? error.message : String(error);
if (errorMsg.includes('Incomplete JSON segment') && attempts <= maxRetries) {
console.warn(`[Gemini] Incomplete JSON segment error, retrying (${attempts}/${maxRetries})...`);
await new Promise(resolve => setTimeout(resolve, 1000 * attempts));
continue;
}
throw error;
}
}
}
async *createResponseStream(messages, model, agent, requestId) {
const { getToolsFromAgent } = await import('../utils/agent.js');
const tools = agent ? await getToolsFromAgent(agent) : [];
const settings = agent?.modelSettings;
let messageId = uuidv4();
let contentBuffer = '';
let thoughtBuffer = '';
let latestThoughtSignature = null;
let eventOrder = 0;
const shownGrounding = new Set();
const withRequestId = (event) => {
return requestId ? { ...event, request_id: requestId } : event;
};
const chunks = [];
try {
const contents = await convertToGeminiContents(model, messages);
if (contents.length === 0) {
console.warn('Gemini API Warning: No valid content found in messages after conversion. Adding default message.');
contents.push({
role: 'user',
parts: [
{
text: "Let's think this through step by step.",
},
],
});
}
const lastContent = contents[contents.length - 1];
if (lastContent.role !== 'user') {
console.warn("Last message in history is not from 'user'. Gemini might not respond as expected.");
}
let thinkingBudget = null;
for (const [suffix, budget] of Object.entries(THINKING_BUDGET_CONFIGS)) {
if (model.endsWith(suffix)) {
thinkingBudget = budget;
model = model.slice(0, -suffix.length);
break;
}
}
const config = {
thinkingConfig: {
includeThoughts: true,
},
};
if (thinkingBudget) {
config.thinkingConfig.thinkingBudget = thinkingBudget;
}
if (settings?.stop_sequence) {
config.stopSequences = [settings.stop_sequence];
}
if (settings?.temperature) {
config.temperature = settings.temperature;
}
if (settings?.max_tokens) {
config.maxOutputTokens = settings.max_tokens;
}
if (settings?.top_p) {
config.topP = settings.top_p;
}
if (settings?.top_k) {
config.topK = settings.top_k;
}
if (settings?.json_schema) {
config.responseMimeType = 'application/json';
config.responseSchema = settings.json_schema.schema;
if (config.responseSchema) {
const removeAdditionalProperties = (obj) => {
if (!obj || typeof obj !== 'object') {
return;
}
if ('additionalProperties' in obj) {
delete obj.additionalProperties;
}
if (obj.properties && typeof obj.properties === 'object') {
Object.values(obj.properties).forEach(prop => {
removeAdditionalProperties(prop);
});
}
if (obj.items) {
removeAdditionalProperties(obj.items);
}
['oneOf', 'anyOf', 'allOf'].forEach(key => {
if (obj[key] && Array.isArray(obj[key])) {
obj[key].forEach((subSchema) => {
removeAdditionalProperties(subSchema);
});
}
});
};
removeAdditionalProperties(config.responseSchema);
}
}
let hasGoogleWebSearch = false;
let hasCodeExecutionTool = false;
let functionDeclarations = [];
if (tools && tools.length > 0) {
hasGoogleWebSearch = tools.some(tool => tool.definition.function.name === 'google_web_search');
hasCodeExecutionTool = tools.some(tool => tool.definition.function.name === 'code_execution');
functionDeclarations = await convertToGeminiFunctionDeclarations(tools);
let allowedFunctionNames = [];
if (functionDeclarations.length > 0) {
config.tools = [{ functionDeclarations }];
if (settings?.tool_choice) {
let toolChoice;
if (typeof settings.tool_choice === 'object' &&
settings.tool_choice?.type === 'function' &&
settings.tool_choice?.function?.name) {
toolChoice = FunctionCallingConfigMode.ANY;
allowedFunctionNames = [settings.tool_choice.function.name];
}
else if (settings.tool_choice === 'required') {
toolChoice = FunctionCallingConfigMode.ANY;
}
else if (settings.tool_choice === 'auto') {
toolChoice = FunctionCallingConfigMode.AUTO;
}
else if (settings.tool_choice === 'none') {
toolChoice = FunctionCallingConfigMode.NONE;
}
if (toolChoice) {
config.toolConfig = {
functionCallingConfig: {
mode: toolChoice,
},
};
if (allowedFunctionNames.length > 0) {
config.toolConfig.functionCallingConfig.allowedFunctionNames = allowedFunctionNames;
}
}
}
}
else if (!hasGoogleWebSearch && !hasCodeExecutionTool) {
console.warn('Tools were provided but resulted in empty declarations after conversion.');
}
}
if (hasGoogleWebSearch || hasCodeExecutionTool || functionDeclarations.length > 0) {
const toolGroups = [];
if (hasGoogleWebSearch) {
console.log('[Gemini] Enabling Google Search grounding');
toolGroups.push({ googleSearch: {} });
}
if (hasCodeExecutionTool) {
console.log('[Gemini] Enabling code execution');
toolGroups.push({ codeExecution: {} });
}
if (functionDeclarations.length > 0) {
toolGroups.push({ functionDeclarations });
}
config.tools = toolGroups;
if (functionDeclarations.length === 0) {
delete config.toolConfig;
}
}
const requestParams = {
model,
contents,
config,
};
const loggedRequestId = log_llm_request(agent.agent_id, 'google', model, requestParams, new Date(), requestId, agent.tags);
requestId = loggedRequestId;
const { waitWhilePaused } = await import('../utils/pause_controller.js');
await waitWhilePaused(100, agent.abortSignal);
const getStreamFn = () => this.client.models.generateContentStream(requestParams);
const response = this.retryStreamOnIncompleteJson(getStreamFn);
let usageMetadata;
for await (const chunk of response) {
chunks.push(chunk);
if (chunk.responseId) {
messageId = chunk.responseId;
}
if (isPaused()) {
console.log(`[Gemini] System paused during stream for model ${model}. Waiting...`);
await waitWhilePaused(100, agent.abortSignal);
console.log(`[Gemini] System resumed, continuing stream for model ${model}`);
}
if (chunk.functionCalls && chunk.functionCalls.length > 0) {
const functionCallPartSignatures = [];
for (const candidate of chunk.candidates || []) {
const parts = candidate?.content?.parts || [];
for (const part of parts) {
if (part?.functionCall) {
functionCallPartSignatures.push(normalizeThoughtSignature(part.thoughtSignature || part.thought_signature));
}
}
}
for (const fc of chunk.functionCalls) {
if (fc && fc.name) {
const thoughtSignature = normalizeThoughtSignature(fc.thoughtSignature || fc.thought_signature) ||
functionCallPartSignatures.shift() ||
null;
yield withRequestId({
type: 'tool_start',
tool_call: {
id: fc.id || `call_${uuidv4()}`,
type: 'function',
...(thoughtSignature ? { thought_signature: thoughtSignature } : {}),
function: {
name: fc.name,
arguments: JSON.stringify(fc.args || {}),
},
},
});
}
}
}
for (const candidate of chunk.candidates) {
if (candidate.content?.parts) {
for (const part of candidate.content.parts) {
const thoughtSignature = normalizeThoughtSignature(part.thoughtSignature || part.thought_signature);
if (thoughtSignature) {
latestThoughtSignature = thoughtSignature;
}
let text = '';
if (part.text) {
text += part.text;
}
if (part.executableCode) {
if (text) {
text += '\n\n';
}
text += part.executableCode;
}
if (part.videoMetadata) {
if (text) {
text += '\n\n';
}
text += JSON.stringify(part.videoMetadata);
}
if (text.length > 0) {
const ev = {
type: 'message_delta',
content: '',
message_id: messageId,
order: eventOrder++,
};
if (part.thought) {
thoughtBuffer += text;
ev.thinking_content = text;
}
else {
contentBuffer += text;
ev.content = text;
}
yield ev;
}
if (part.inlineData?.data) {
yield withRequestId({
type: 'file_complete',
data_format: 'base64',
data: part.inlineData.data,
mime_type: part.inlineData.mimeType || 'image/png',
message_id: uuidv4(),
order: eventOrder++,
});
}
}
}
const gChunks = candidate.groundingMetadata?.groundingChunks;
if (Array.isArray(gChunks)) {
const newChunks = gChunks.filter(c => c?.web?.uri && !shownGrounding.has(c.web.uri));
if (newChunks.length) {
newChunks.forEach(c => shownGrounding.add(c.web.uri));
const formatted = formatGroundingChunks(newChunks);
yield withRequestId({
type: 'message_delta',
content: '\n\nSearch Results:\n' + formatted + '\n',
message_id: messageId,
order: eventOrder++,
});
contentBuffer += '\n\nSearch Results:\n' + formatted + '\n';
}
}
}
if (chunk.usageMetadata) {
usageMetadata = chunk.usageMetadata;
}
}
if (usageMetadata) {
const calculatedUsage = costTracker.addUsage({
model,
input_tokens: usageMetadata.promptTokenCount || 0,
output_tokens: usageMetadata.candidatesTokenCount || 0,
cached_tokens: usageMetadata.cachedContentTokenCount || 0,
metadata: {
total_tokens: usageMetadata.totalTokenCount || 0,
reasoning_tokens: usageMetadata.thoughtsTokenCount || 0,
tool_tokens: usageMetadata.toolUsePromptTokenCount || 0,
},
});
if (!hasEventHandler()) {
yield withRequestId({
type: 'cost_update',
usage: {
...calculatedUsage,
total_tokens: usageMetadata.totalTokenCount || 0,
},
});
}
}
else {
console.warn('[Gemini] No usage metadata found in the response. Using token estimation.');
let inputText = '';
for (const content of contents) {
if (content.parts) {
for (const part of content.parts) {
if (part.text) {
inputText += part.text + '\n';
}
}
}
}
const calculatedUsage = costTracker.addEstimatedUsage(model, inputText, contentBuffer + thoughtBuffer, {
provider: 'gemini',
});
if (!hasEventHandler()) {
yield withRequestId({
type: 'cost_update',
usage: {
...calculatedUsage,
total_tokens: calculatedUsage.input_tokens + calculatedUsage.output_tokens,
},
});
}
}
if (contentBuffer || thoughtBuffer) {
yield withRequestId({
type: 'message_complete',
content: contentBuffer,
thinking_content: thoughtBuffer,
...(latestThoughtSignature ? { thinking_signature: latestThoughtSignature } : {}),
message_id: messageId,
});
}
}
catch (error) {
log_llm_error(requestId, error);
const errorMessage = error instanceof Error ? error.stack || error.message : String(error);
if (errorMessage.includes('Incomplete JSON segment')) {
console.error('[Gemini] Stream terminated with incomplete JSON. This may indicate network issues or timeouts.');
}
console.error('\n=== Gemini error ===');
console.dir(error, { depth: null });
console.error('\n=== JSON dump of error ===');
console.error(truncateLargeValues(JSON.stringify(error, Object.getOwnPropertyNames(error), 2)));
console.error('\n=== Manual property walk ===');
for (const key of Reflect.ownKeys(error)) {
console.error(`${String(key)}:`, truncateLargeValues(error[key]));
}
yield withRequestId({
type: 'error',
error: `Gemini error ${model}: ${errorMessage}`,
});
if (contentBuffer || thoughtBuffer) {
yield withRequestId({
type: 'message_complete',
content: contentBuffer,
thinking_content: thoughtBuffer,
...(latestThoughtSignature ? { thinking_signature: latestThoughtSignature } : {}),
message_id: messageId,
});
}
}
finally {
log_llm_response(requestId, chunks);
}
}
async createImage(prompt, model, agent, opts) {
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
let finalRequestId = requestId;
try {
model = model || 'gemini-2.5-flash-image-preview';
const numberOfImages = opts?.n || 1;
const { getToolsFromAgent } = await import('../utils/agent.js');
const tools = agent ? await getToolsFromAgent(agent) : [];
const hasGoogleWebSearch = tools?.some(tool => tool.definition.function.name === 'google_web_search');
const hasOtherTools = tools?.some(tool => tool.definition.function.name !== 'google_web_search');
if (hasOtherTools) {
console.warn('[Gemini] Image generation ignores function tools; only google_web_search is supported.');
}
const explicitWebGrounding = opts?.grounding?.web_search;
const explicitImageGrounding = opts?.grounding?.image_search;
const enableWebGrounding = explicitWebGrounding ?? hasGoogleWebSearch ?? false;
const isGemini31FlashImageModel = model.includes('gemini-3.1-flash-image-preview');
const enableImageGrounding = explicitImageGrounding === true && isGemini31FlashImageModel;
if (explicitImageGrounding && !isGemini31FlashImageModel) {
console.warn('[Gemini] Image Search grounding is only available for gemini-3.1-flash-image-preview. Ignoring image_search=true.');
}
const thinkingOptions = opts?.thinking;
const hasThinkingOptionsObject = thinkingOptions !== null &&
typeof thinkingOptions === 'object' &&
!Array.isArray(thinkingOptions);
const includeThoughts = hasThinkingOptionsObject && thinkingOptions.include_thoughts === true;
const requestedThinkingLevel = hasThinkingOptionsObject
? thinkingOptions.level
: undefined;
const thinkingLevel = requestedThinkingLevel === 'high' ? 'High' : requestedThinkingLevel ? 'Minimal' : undefined;
if (requestedThinkingLevel && !isGemini31FlashImageModel) {
console.warn('[Gemini] thinking.level is currently supported for gemini-3.1-flash-image-preview only. Ignoring thinking level.');
}
if (hasThinkingOptionsObject && 'include_thoughts' in thinkingOptions && !isGemini31FlashImageModel) {
console.warn('[Gemini] thinking.include_thoughts is currently supported for gemini-3.1-flash-image-preview only. Ignoring include_thoughts.');
}
let aspectRatio = '1:1';
if (opts?.size === 'landscape')
aspectRatio = '16:9';
else if (opts?.size === 'portrait')
aspectRatio = '9:16';
console.log(`[Gemini] Generating ${numberOfImages} image(s) with model ${model}, prompt: "${prompt.substring(0, 100)}${prompt.length > 100 ? '...' : ''}"`);
if (model.includes('gemini-2.5-flash-image-preview') ||
model.includes('gemini-3.1-flash-image-preview') ||
model.includes('gemini-3-pro-image-preview')) {
let aggregateMetadata = { model };
const sizeMap = {
'1:1': { ar: '1:1' },
'1:4': { ar: '1:4' },
'1:8': { ar: '1:8' },
'2:3': { ar: '2:3' },
'3:2': { ar: '3:2' },
'3:4': { ar: '3:4' },
'4:1': { ar: '4:1' },
'4:3': { ar: '4:3' },
'4:5': { ar: '4:5' },
'5:4': { ar: '5:4' },
'8:1': { ar: '8:1' },
'9:16': { ar: '9:16' },
'16:9': { ar: '16:9' },
'21:9': { ar: '21:9' },
square: { ar: '1:1' },
landscape: { ar: '16:9' },
portrait: { ar: '9:16' },
'256x256': { ar: '1:1' },
'512x512': { ar: '1:1' },
'1024x1024': { ar: '1:1' },
'1536x1024': { ar: '3:2' },
'1024x1536': { ar: '2:3' },
'1696x2528': { ar: '2:3' },
'2048x2048': { ar: '1:1' },
'1792x1024': { ar: '16:9' },
'1024x1