UNPKG

@jmndao/mongoose-ai

Version:

AI-powered Mongoose plugin for intelligent document processing with auto-summarization, semantic search, MongoDB Vector Search, and function calling

1,613 lines (1,598 loc) 51.2 kB
// src/config-helpers.ts var DEFAULT_CONFIG = { advanced: { maxRetries: 2, timeout: 3e4, skipOnUpdate: false, forceRegenerate: false, logLevel: "warn", continueOnError: true, enableFunctions: false }, openai: { chatModel: "gpt-3.5-turbo", embeddingModel: "text-embedding-3-small", maxTokens: 200, temperature: 0.3 }, anthropic: { chatModel: "claude-3-haiku-20240307", maxTokens: 200, temperature: 0.3 }, ollama: { chatModel: "llama3.2", embeddingModel: "nomic-embed-text", maxTokens: 200, temperature: 0.3, endpoint: "http://localhost:11434" }, vectorSearch: { enabled: true, indexName: "vector_index", autoCreateIndex: true, similarity: "cosine" } }; function validateApiKey(apiKey, provider) { if (typeof apiKey !== "string" || apiKey.length < 2) { return false; } switch (provider) { case "openai": return apiKey.startsWith("sk-") && apiKey.length > 20; // More strict validation case "anthropic": return apiKey.startsWith("sk-ant-") && apiKey.length > 30 || apiKey.length > 20; case "ollama": return apiKey.length >= 2; // Ollama just needs non-empty string default: return false; } } function createAdvancedAIConfig(options) { if (!validateApiKey(options.apiKey, options.provider)) { throw new Error(`Invalid API key format for ${options.provider}`); } const defaultModelConfig = options.provider === "openai" ? DEFAULT_CONFIG.openai : options.provider === "anthropic" ? DEFAULT_CONFIG.anthropic : DEFAULT_CONFIG.ollama; return { model: options.model, provider: options.provider, field: options.field, credentials: { apiKey: options.apiKey }, prompt: options.prompt, advanced: { ...DEFAULT_CONFIG.advanced, ...options.advanced }, modelConfig: { ...defaultModelConfig, ...options.modelConfig }, vectorSearch: { ...DEFAULT_CONFIG.vectorSearch, ...options.vectorSearch }, includeFields: options.includeFields, excludeFields: options.excludeFields, functions: options.functions }; } function createAIConfig(options) { return createAdvancedAIConfig({ ...options, provider: "openai" // Default to OpenAI for backward compatibility }); } function createOllamaConfig(options) { return createAdvancedAIConfig({ ...options, apiKey: "local", // Placeholder for Ollama provider: "ollama", modelConfig: { endpoint: options.endpoint, chatModel: options.chatModel, embeddingModel: options.embeddingModel } }); } function estimateTokenCount(text) { if (!text || typeof text !== "string") return 0; return Math.ceil(text.length / 4); } function estimateCost(tokenCount, model, provider = "openai") { const pricing = { openai: { "gpt-3.5-turbo": 15e-4, "gpt-4": 0.03, "gpt-4o": 5e-3, "gpt-4o-mini": 15e-5, "text-embedding-3-small": 2e-5, "text-embedding-3-large": 13e-5 }, anthropic: { "claude-3-haiku-20240307": 25e-5, "claude-3-sonnet-20240229": 3e-3, "claude-3-opus-20240229": 0.015 }, ollama: { // Local models have no API costs "llama3.2": 0, llama2: 0, mistral: 0, "nomic-embed-text": 0 } }; const providerPricing = pricing[provider]; const pricePerToken = providerPricing?.[model] || 0; return tokenCount / 1e3 * pricePerToken; } function checkEnvironment() { const missing = []; const warnings = []; const isNode = typeof globalThis !== "undefined" && typeof globalThis.process !== "undefined" && globalThis.process.versions?.node; if (!isNode) { warnings.push("Not running in Node.js environment"); return { isValid: false, missing, warnings }; } const env = globalThis.process.env; if (!env.OPENAI_API_KEY && !env.ANTHROPIC_API_KEY) { missing.push( "OPENAI_API_KEY, ANTHROPIC_API_KEY environment variable, or use Ollama for local processing" ); } if (env.OPENAI_API_KEY && !validateApiKey(env.OPENAI_API_KEY, "openai")) { warnings.push("OPENAI_API_KEY format appears invalid"); } if (env.ANTHROPIC_API_KEY && !validateApiKey(env.ANTHROPIC_API_KEY, "anthropic")) { warnings.push("ANTHROPIC_API_KEY format appears invalid"); } return { isValid: missing.length === 0, missing, warnings }; } // src/plugin.ts import { Schema } from "mongoose"; // src/providers/openai.ts import OpenAI from "openai"; // src/providers/base.ts var BaseProvider = class { constructor(credentials, advancedOptions = {}) { this.validateCredentials(credentials); this.advanced = { maxRetries: advancedOptions.maxRetries ?? 2, timeout: advancedOptions.timeout ?? 3e4, skipOnUpdate: advancedOptions.skipOnUpdate ?? false, forceRegenerate: advancedOptions.forceRegenerate ?? false, logLevel: advancedOptions.logLevel || "warn", continueOnError: advancedOptions.continueOnError ?? true, enableFunctions: advancedOptions.enableFunctions ?? false }; } /** * Execute functions with error handling */ async executeFunctions(functions, functionCalls, document) { const results = []; for (const call of functionCalls) { const func = functions.find((f) => f.name === call.name); if (!func) { results.push({ name: call.name, success: false, error: "Function not found", executedAt: /* @__PURE__ */ new Date() }); continue; } try { const result = await func.handler(call.arguments || {}, document); results.push({ name: call.name, success: true, result: call.arguments, // Store the arguments that were applied executedAt: /* @__PURE__ */ new Date() }); this.log( "debug", `Function ${call.name} executed successfully. Document updated.` ); } catch (error) { results.push({ name: call.name, success: false, error: error instanceof Error ? error.message : "Unknown error", executedAt: /* @__PURE__ */ new Date() }); this.log("error", `Function ${call.name} failed:`, error); } } return results; } /** * Convert document to clean text */ prepareText(document) { if (!document || typeof document !== "object") { return ""; } const clean = { ...document }; const systemFields = ["_id", "__v", "createdAt", "updatedAt", "id"]; systemFields.forEach((field) => delete clean[field]); try { return JSON.stringify(clean, null, 2).replace(/[{}",[\]]/g, " ").replace(/\s+/g, " ").trim(); } catch (error) { this.log("warn", "Failed to stringify document, using fallback"); return String(document).trim(); } } /** * Truncate text to specified length */ truncateText(text, maxLength) { if (!text || typeof text !== "string") { return ""; } if (text.length <= maxLength) return text; const truncated = text.substring(0, maxLength); const lastSpace = truncated.lastIndexOf(" "); return lastSpace > maxLength * 0.8 ? truncated.substring(0, lastSpace) + "..." : truncated + "..."; } /** * Extract error message */ getErrorMessage(error) { if (typeof error === "string") return error; if (error?.message) return error.message; if (error?.error?.message) return error.error.message; if (error?.response?.data?.error?.message) return error.response.data.error.message; return "Unknown error occurred"; } /** * Log messages based on level */ log(level, message, error) { const levels = { debug: 0, info: 1, warn: 2, error: 3 }; const currentLevel = levels[this.advanced.logLevel]; if (levels[level] >= currentLevel) { const timestamp = (/* @__PURE__ */ new Date()).toISOString(); const prefix = `[${timestamp}] [mongoose-ai] [${level.toUpperCase()}]`; if (error && level === "error") { console[level](`${prefix} ${message}`, error); } else if (level === "debug" && typeof error !== "undefined") { console[level](`${prefix} ${message}`, error); } else { console[level](`${prefix} ${message}`); } } } }; // src/providers/openai.ts var OpenAIProvider = class extends BaseProvider { constructor(credentials, modelConfig = {}, advancedOptions = {}) { super(credentials, advancedOptions); this.config = { chatModel: modelConfig.chatModel || "gpt-3.5-turbo", embeddingModel: modelConfig.embeddingModel || "text-embedding-3-small", maxTokens: modelConfig.maxTokens || 200, temperature: modelConfig.temperature || 0.3 }; try { this.client = new OpenAI({ apiKey: credentials.apiKey, organization: credentials.organizationId, timeout: this.advanced.timeout, maxRetries: this.advanced.maxRetries }); this.log("info", "OpenAI provider initialized successfully"); } catch (error) { throw new Error( `Failed to initialize OpenAI client: ${this.getErrorMessage(error)}` ); } } /** * Generate summary for document with optional function calling */ async summarize(document, customPrompt, functions) { const startTime = Date.now(); try { const text = this.prepareText(document); if (!text.trim()) { throw new Error("No content to summarize"); } const prompt = customPrompt || "Summarize this content in 2-3 clear sentences:"; const tools = this.prepareFunctionTools(functions); const requestParams = { model: this.config.chatModel, messages: [ { role: "system", content: prompt }, { role: "user", content: text } ], max_tokens: this.config.maxTokens, temperature: this.config.temperature }; if (tools.length > 0 && this.advanced.enableFunctions) { requestParams.tools = tools; requestParams.tool_choice = "auto"; } const response = await this.client.chat.completions.create(requestParams); const choice = response.choices[0]; if (!choice) { throw new Error("No response choice received from OpenAI"); } let summary = choice.message?.content?.trim() || ""; let functionResults = []; const toolCalls = choice.message?.tool_calls; if (toolCalls && toolCalls.length > 0 && functions && this.advanced.enableFunctions) { functionResults = await this.executeFunctions( functions, toolCalls.map((call) => ({ name: call.function.name, arguments: JSON.parse(call.function.arguments || "{}") })), document ); } if (!summary && functionResults.length > 0) { summary = "Analysis completed with automated classifications applied."; this.log( "info", "No summary content provided, using default summary due to tool calls" ); } if (!summary) { throw new Error("Failed to generate summary - no content returned"); } const processingTime = Date.now() - startTime; this.log("info", `Summary generated in ${processingTime}ms`); return { summary, generatedAt: /* @__PURE__ */ new Date(), model: this.config.chatModel, tokenCount: response.usage?.total_tokens, processingTime, ...functionResults.length > 0 && { functionResults } }; } catch (error) { const processingTime = Date.now() - startTime; this.log( "error", `Summary generation failed after ${processingTime}ms:`, error ); throw new Error( `Summary generation failed: ${this.getErrorMessage(error)}` ); } } /** * Generate embedding for text */ async generateEmbedding(text) { const startTime = Date.now(); try { if (!text || typeof text !== "string") { throw new Error("Text is required for embedding"); } const processedText = this.truncateText(text, 8e3); const response = await this.client.embeddings.create({ model: this.config.embeddingModel, input: processedText }); const embedding = response.data[0]?.embedding; if (!embedding || !Array.isArray(embedding)) { throw new Error( "Failed to generate embedding - no embedding data returned" ); } const processingTime = Date.now() - startTime; this.log("info", `Embedding generated in ${processingTime}ms`); return { embedding, generatedAt: /* @__PURE__ */ new Date(), model: this.config.embeddingModel, dimensions: embedding.length, processingTime }; } catch (error) { const processingTime = Date.now() - startTime; this.log( "error", `Embedding generation failed after ${processingTime}ms:`, error ); throw new Error( `Embedding generation failed: ${this.getErrorMessage(error)}` ); } } /** * Prepare function tools for OpenAI API */ prepareFunctionTools(functions) { if (!functions || !this.advanced.enableFunctions) { return []; } return functions.map((func) => { const cleanParameters = {}; const requiredFields = []; Object.entries(func.parameters).forEach(([key, param]) => { if (param.required === true) { requiredFields.push(key); } const { required, ...cleanParam } = param; cleanParameters[key] = cleanParam; }); return { type: "function", function: { name: func.name, description: func.description, parameters: { type: "object", properties: cleanParameters, required: requiredFields } } }; }); } /** * Validate credentials */ validateCredentials(credentials) { if (!credentials || typeof credentials !== "object") { throw new Error("Credentials object is required"); } if (!credentials.apiKey || typeof credentials.apiKey !== "string") { throw new Error("OpenAI API key is required"); } if (!credentials.apiKey.startsWith("sk-")) { throw new Error('Invalid OpenAI API key format - must start with "sk-"'); } if (credentials.apiKey.length < 20) { throw new Error("OpenAI API key appears to be invalid - too short"); } } /** * Get provider information */ getProviderInfo() { return { name: "OpenAI", version: "1.1.0", models: this.config, advanced: this.advanced }; } }; // src/providers/anthropic.ts var AnthropicProvider = class extends BaseProvider { constructor(credentials, modelConfig = {}, advancedOptions = {}) { super(credentials, advancedOptions); this.apiKey = credentials.apiKey; this.config = { chatModel: modelConfig.chatModel || "claude-3-haiku-20240307", maxTokens: modelConfig.maxTokens || 200, temperature: modelConfig.temperature || 0.3 }; this.log("info", "Anthropic provider initialized successfully"); } /** * Generate summary for document with optional function calling */ async summarize(document, customPrompt, functions) { const startTime = Date.now(); try { const text = this.prepareText(document); if (!text.trim()) { throw new Error("No content to summarize"); } const prompt = customPrompt || "Summarize this content in 2-3 clear sentences:"; const tools = this.prepareFunctionTools(functions); const requestBody = { model: this.config.chatModel, max_tokens: this.config.maxTokens, temperature: this.config.temperature, messages: [ { role: "user", content: `${prompt} Content to analyze: ${text}` } ] }; if (tools.length > 0 && this.advanced.enableFunctions) { requestBody.tools = tools; } const response = await this.makeAnthropicRequest(requestBody); const textContent = response.content?.find((c) => c.type === "text"); let summary = textContent?.text?.trim() || ""; const functionResults = []; const toolUse = response.content?.filter((c) => c.type === "tool_use") || []; if (toolUse.length > 0 && functions && this.advanced.enableFunctions) { for (const use of toolUse) { try { const func = functions.find((f) => f.name === use.name); if (!func) { functionResults.push({ name: use.name, success: false, error: "Function not found", executedAt: /* @__PURE__ */ new Date() }); continue; } const docInstance = document; await func.handler(use.input || {}, docInstance); functionResults.push({ name: use.name, success: true, result: use.input, executedAt: /* @__PURE__ */ new Date() }); this.log( "debug", `Function ${use.name} executed successfully. Document updated.` ); } catch (error) { functionResults.push({ name: use.name, success: false, error: error instanceof Error ? error.message : "Unknown error", executedAt: /* @__PURE__ */ new Date() }); this.log("error", `Function ${use.name} failed:`, error); } } if (!summary && functionResults.length > 0) { summary = "Analysis completed with automated classifications applied."; this.log( "info", "No summary content provided, using default summary due to tool calls" ); } } if (!summary) { throw new Error("Failed to generate summary - no content returned"); } const processingTime = Date.now() - startTime; this.log("info", `Summary generated in ${processingTime}ms`); return { summary, generatedAt: /* @__PURE__ */ new Date(), model: this.config.chatModel, tokenCount: response.usage?.input_tokens + response.usage?.output_tokens, processingTime, ...functionResults.length > 0 && { functionResults } }; } catch (error) { const processingTime = Date.now() - startTime; this.log( "error", `Summary generation failed after ${processingTime}ms:`, error ); throw new Error( `Summary generation failed: ${this.getErrorMessage(error)}` ); } } /** * Generate embedding for text - Note: Anthropic doesn't provide embeddings */ async generateEmbedding(text) { throw new Error( "Anthropic provider does not support embeddings. Use OpenAI provider for embedding models." ); } /** * Make API request to Anthropic */ async makeAnthropicRequest(body) { try { const response = await fetch("https://api.anthropic.com/v1/messages", { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": this.apiKey, "anthropic-version": "2023-06-01" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.advanced.timeout) }); if (!response.ok) { const errorText = await response.text(); let errorData = {}; try { errorData = JSON.parse(errorText); } catch { errorData = { error: { message: errorText } }; } const errorMessage = errorData.error?.message || errorData.message || `HTTP ${response.status} ${response.statusText}`; throw new Error(`Anthropic API error: ${errorMessage}`); } return await response.json(); } catch (error) { if (error instanceof Error) { throw error; } throw new Error(`Request failed: ${String(error)}`); } } /** * Prepare function tools for Anthropic API */ prepareFunctionTools(functions) { if (!functions || !this.advanced.enableFunctions) { return []; } return functions.map((func) => { const cleanProperties = {}; const requiredFields = []; Object.entries(func.parameters).forEach(([key, param]) => { if (param.required === true) { requiredFields.push(key); } const { required, ...cleanParam } = param; cleanProperties[key] = cleanParam; }); return { name: func.name, description: func.description, input_schema: { type: "object", properties: cleanProperties, required: requiredFields } }; }); } /** * Validate credentials */ validateCredentials(credentials) { if (!credentials || typeof credentials !== "object") { throw new Error("Credentials object is required"); } if (!credentials.apiKey || typeof credentials.apiKey !== "string") { throw new Error("Anthropic API key is required"); } if (credentials.apiKey.length < 20) { throw new Error("Anthropic API key appears to be invalid - too short"); } if (!credentials.apiKey.startsWith("sk-ant-") && credentials.apiKey.length < 40) { this.log( "warn", "Anthropic API key format may be invalid - expected to start with 'sk-ant-'" ); } } /** * Get provider information */ getProviderInfo() { return { name: "Anthropic", version: "1.1.0", models: this.config, advanced: this.advanced }; } }; // src/providers/ollama.ts var OllamaProvider = class extends BaseProvider { constructor(credentials, modelConfig = {}, advancedOptions = {}) { super(credentials, advancedOptions); this.config = { chatModel: modelConfig.chatModel || "llama3.2", embeddingModel: modelConfig.embeddingModel || "nomic-embed-text", maxTokens: modelConfig.maxTokens || 200, temperature: modelConfig.temperature || 0.3, endpoint: modelConfig.endpoint || "http://localhost:11434" }; this.endpoint = this.config.endpoint; this.log("info", "Ollama provider initialized successfully"); } /** * Generate summary for document with optional function calling */ async summarize(document, customPrompt, functions) { const startTime = Date.now(); try { const text = this.prepareText(document); if (!text.trim()) { throw new Error("No content to summarize"); } const prompt = customPrompt || "Summarize this content in 2-3 clear sentences:"; let fullPrompt = `${prompt} Content to analyze: ${text}`; let functionResults = []; if (functions && functions.length > 0 && this.advanced.enableFunctions) { const functionInstructions = this.prepareFunctionInstructions(functions); fullPrompt += ` ${functionInstructions}`; } const requestBody = { model: this.config.chatModel, prompt: fullPrompt, options: { num_predict: this.config.maxTokens, temperature: this.config.temperature }, stream: false }; const response = await this.makeOllamaRequest( "/api/generate", requestBody ); let summary = response.response?.trim() || ""; if (functions && functions.length > 0 && this.advanced.enableFunctions) { functionResults = await this.parseFunctionCalls( summary, functions, document ); if (!summary && functionResults.length > 0) { summary = "Analysis completed with automated classifications applied."; this.log( "info", "No summary content provided, using default summary due to function calls" ); } } if (!summary) { throw new Error("Failed to generate summary - no content returned"); } const processingTime = Date.now() - startTime; this.log("info", `Summary generated in ${processingTime}ms`); return { summary, generatedAt: /* @__PURE__ */ new Date(), model: this.config.chatModel, tokenCount: this.estimateTokenCount(summary), processingTime, ...functionResults.length > 0 && { functionResults } }; } catch (error) { const processingTime = Date.now() - startTime; this.log( "error", `Summary generation failed after ${processingTime}ms:`, error ); throw new Error( `Summary generation failed: ${this.getErrorMessage(error)}` ); } } /** * Generate embedding for text */ async generateEmbedding(text) { const startTime = Date.now(); try { if (!text || typeof text !== "string") { throw new Error("Text is required for embedding"); } const processedText = this.truncateText(text, 8e3); const requestBody = { model: this.config.embeddingModel, prompt: processedText }; const response = await this.makeOllamaRequest( "/api/embeddings", requestBody ); const embedding = response.embedding; if (!embedding || !Array.isArray(embedding)) { throw new Error( "Failed to generate embedding - no embedding data returned" ); } const processingTime = Date.now() - startTime; this.log("info", `Embedding generated in ${processingTime}ms`); return { embedding, generatedAt: /* @__PURE__ */ new Date(), model: this.config.embeddingModel, dimensions: embedding.length, processingTime }; } catch (error) { const processingTime = Date.now() - startTime; this.log( "error", `Embedding generation failed after ${processingTime}ms:`, error ); throw new Error( `Embedding generation failed: ${this.getErrorMessage(error)}` ); } } /** * Make API request to Ollama */ async makeOllamaRequest(endpoint, body) { try { const url = `${this.endpoint}${endpoint}`; const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.advanced.timeout) }); if (!response.ok) { const errorText = await response.text(); let errorData = {}; try { errorData = JSON.parse(errorText); } catch { errorData = { error: { message: errorText } }; } const errorMessage = errorData.error?.message || errorData.message || `HTTP ${response.status} ${response.statusText}`; throw new Error(`Ollama API error: ${errorMessage}`); } return await response.json(); } catch (error) { if (error instanceof Error) { if (error.name === "AbortError") { throw new Error("Ollama request timeout - is Ollama running?"); } if (error.message.includes("fetch")) { throw new Error( "Cannot connect to Ollama - is it running on " + this.endpoint + "?" ); } throw error; } throw new Error(`Request failed: ${String(error)}`); } } /** * Prepare function calling instructions for Ollama */ prepareFunctionInstructions(functions) { let instructions = "\nAvailable actions you can perform:\n"; functions.forEach((func, index) => { instructions += `${index + 1}. ${func.name}: ${func.description} `; Object.entries(func.parameters).forEach(([key, param]) => { instructions += ` - ${key}: ${param.description}`; if (param.enum) { instructions += ` (options: ${param.enum.join(", ")})`; } instructions += "\n"; }); }); instructions += "\nAfter your summary, if appropriate, specify actions using this format:\n"; instructions += "ACTIONS:\n"; instructions += "- function_name: parameter_value\n"; instructions += "- another_function: another_value\n"; return instructions; } /** * Parse function calls from Ollama response */ async parseFunctionCalls(response, functions, document) { const results = []; try { const actionsMatch = response.match(/ACTIONS:\s*([\s\S]*?)(?:\n\n|$)/i); if (!actionsMatch) { return results; } const actionsText = actionsMatch[1]; const actionLines = actionsText.split("\n").filter((line) => line.trim().startsWith("-")).map((line) => line.replace(/^-\s*/, "").trim()); for (const actionLine of actionLines) { const [functionName, ...valueParts] = actionLine.split(":"); const value = valueParts.join(":").trim(); const func = functions.find((f) => f.name === functionName.trim()); if (!func) { results.push({ name: functionName.trim(), success: false, error: "Function not found", executedAt: /* @__PURE__ */ new Date() }); continue; } try { const args = this.parseSimpleArgs(func, value); await func.handler(args, document); results.push({ name: func.name, success: true, result: args, executedAt: /* @__PURE__ */ new Date() }); this.log("debug", `Function ${func.name} executed successfully`); } catch (error) { results.push({ name: func.name, success: false, error: error instanceof Error ? error.message : "Unknown error", executedAt: /* @__PURE__ */ new Date() }); this.log("error", `Function ${func.name} failed:`, error); } } } catch (error) { this.log("error", "Function parsing failed:", error); } return results; } /** * Parse simple function arguments */ parseSimpleArgs(func, value) { const paramKeys = Object.keys(func.parameters); if (paramKeys.length === 1) { const paramKey = paramKeys[0]; const param = func.parameters[paramKey]; if (param.type === "number") { return { [paramKey]: parseFloat(value) }; } else if (param.type === "array") { return { [paramKey]: value.split(",").map((v) => v.trim()) }; } else { return { [paramKey]: value }; } } try { return JSON.parse(value); } catch { return { [paramKeys[0]]: value }; } } /** * Estimate token count (rough approximation) */ estimateTokenCount(text) { return Math.ceil(text.length / 4); } /** * Validate credentials */ validateCredentials(credentials) { if (!credentials || typeof credentials !== "object") { throw new Error("Credentials object is required"); } if (!credentials.apiKey) { credentials.apiKey = "local"; } } /** * Get provider information */ getProviderInfo() { return { name: "Ollama", version: "1.4.0", models: this.config, advanced: this.advanced, endpoint: this.endpoint }; } }; // src/providers/factory.ts function createProvider(provider, credentials, modelConfig, advancedOptions) { switch (provider) { case "openai": return new OpenAIProvider( credentials, modelConfig, advancedOptions ); case "anthropic": return new AnthropicProvider( credentials, modelConfig, advancedOptions ); case "ollama": return new OllamaProvider( credentials, modelConfig, advancedOptions ); default: throw new Error(`Unsupported provider: ${provider}`); } } function validateProviderModel(provider, model) { const supportedModels = { openai: ["summary", "embedding"], anthropic: ["summary"], // Anthropic doesn't support embeddings ollama: ["summary", "embedding"] // Ollama supports both }; const supported = supportedModels[provider]; if (!supported || !supported.includes(model)) { throw new Error( `Provider '${provider}' does not support model '${model}'. Supported models: ${supported?.join(", ") || "none"}` ); } } // src/utils/vector-search.ts async function detectVectorSearchSupport(model) { try { const collection = model.collection; const db = collection.db; if (!db) { return false; } try { const buildInfo = await db.admin().buildInfo(); const version = buildInfo.version; const majorVersion = parseInt(version.split(".")[0]); if (majorVersion < 6) { return false; } } catch (adminError) { } await model.aggregate([ { $vectorSearch: { index: "test_detection_index", path: "test_field", queryVector: [0.1], numCandidates: 1, limit: 1 } }, { $limit: 0 } // Don't return any results, just test if the operation is supported ]).exec(); return true; } catch (error) { return false; } } async function createVectorIndex(model, embeddingField, dimensions, config) { try { const collection = model.collection; const indexName = config.indexName || "vector_index"; if (typeof collection.listSearchIndexes !== "function") { console.warn( "MongoDB Atlas Search is not available. Vector search index creation skipped." ); return; } const indexes = await collection.listSearchIndexes().toArray(); const existingIndex = indexes.find((idx) => idx.name === indexName); if (existingIndex) { console.log(`Vector search index '${indexName}' already exists`); return; } if (!config.autoCreateIndex) { console.warn( `Vector search index '${indexName}' does not exist and auto-creation is disabled` ); return; } const indexDefinition = { name: indexName, definition: { fields: [ { type: "vector", path: `${embeddingField}.embedding`, numDimensions: dimensions, similarity: config.similarity || "cosine" } ] } }; await collection.createSearchIndex(indexDefinition); console.log( `Created vector search index '${indexName}' for field '${embeddingField}.embedding'` ); console.log( `Index '${indexName}' is being built. It may take 1-2 minutes to become available.` ); } catch (error) { console.error( `Failed to create vector search index: ${error instanceof Error ? error.message : "Unknown error"}` ); } } async function performVectorSearch(model, queryEmbedding, fieldName, options) { const { limit = 10, threshold = 0.7, filter = {}, indexName = "vector_index", numCandidates = limit * 10 } = options; try { const pipeline = [ { $vectorSearch: { index: indexName, path: `${fieldName}.embedding`, queryVector: queryEmbedding, numCandidates: Math.max(numCandidates, limit), limit } } ]; pipeline.push({ $addFields: { similarity: { $meta: "vectorSearchScore" } } }); if (threshold > 0) { pipeline.push({ $match: { similarity: { $gte: threshold } } }); } if (Object.keys(filter).length > 0) { pipeline.push({ $match: filter }); } pipeline.push({ $sort: { similarity: -1 } }); pipeline.push({ $limit: limit }); const results = await model.aggregate(pipeline).exec(); return results.map((doc) => ({ document: doc, similarity: doc.similarity || 0, metadata: { field: fieldName, distance: 1 - (doc.similarity || 0) } })); } catch (error) { console.error( `Vector search failed: ${error instanceof Error ? error.message : "Unknown error"}` ); throw error; } } function cosineSimilarity(a, b) { if (!a || !b || a.length !== b.length) return 0; let dotProduct = 0; let normA = 0; let normB = 0; for (let i = 0; i < a.length; i++) { dotProduct += a[i] * b[i]; normA += a[i] * a[i]; normB += b[i] * b[i]; } const denominator = Math.sqrt(normA) * Math.sqrt(normB); return denominator === 0 ? 0 : dotProduct / denominator; } async function performInMemorySearch(model, queryEmbedding, fieldName, options) { const { limit = 10, threshold = 0.7, filter = {} } = options; try { const searchFilter = { ...filter, [`${fieldName}.embedding`]: { $exists: true } }; const docs = await model.find(searchFilter).exec(); const results = []; for (const doc of docs) { const docEmbedding = doc[fieldName]?.embedding; if (!docEmbedding) continue; const similarity = cosineSimilarity(queryEmbedding, docEmbedding); if (similarity >= threshold) { results.push({ document: doc, similarity, metadata: { field: fieldName, distance: 1 - similarity } }); } } return results.sort((a, b) => b.similarity - a.similarity).slice(0, limit); } catch (error) { console.error( `In-memory search failed: ${error instanceof Error ? error.message : "Unknown error"}` ); throw error; } } // src/plugin.ts var vectorSearchCache = /* @__PURE__ */ new WeakMap(); function aiPlugin(schema, options) { const config = options.ai; if (!config.model || !["summary", "embedding"].includes(config.model)) { throw new Error("Valid model (summary|embedding) required"); } if (!config.provider || !["openai", "anthropic", "ollama"].includes(config.provider)) { throw new Error("Valid provider (openai|anthropic|ollama) required"); } if (!config.field || typeof config.field !== "string" || config.field.trim() === "") { throw new Error("Field name required"); } if (!config.credentials?.apiKey || typeof config.credentials.apiKey !== "string" || config.credentials.apiKey.trim() === "") { throw new Error("API key required"); } validateProviderModel(config.provider, config.model); if (schema.paths[config.field] || schema.virtuals[config.field]) { throw new Error(`Field "${config.field}" already exists in schema`); } let provider; try { provider = createProvider( config.provider, config.credentials, config.modelConfig, config.advanced ); } catch (error) { throw new Error( `Failed to initialize AI provider: ${error instanceof Error ? error.message : "Unknown error"}` ); } const vectorSearchConfig = { enabled: true, indexName: "vector_index", autoCreateIndex: true, similarity: "cosine", ...config.vectorSearch }; if (config.model === "summary") { schema.add({ [config.field]: { type: { summary: { type: String }, generatedAt: { type: Date }, model: { type: String }, tokenCount: { type: Number }, processingTime: { type: Number }, functionResults: [ { name: { type: String }, success: { type: Boolean }, result: { type: Schema.Types.Mixed }, error: { type: String }, executedAt: { type: Date } } ] }, default: void 0 } }); } else if (config.model === "embedding") { schema.add({ [config.field]: { type: { embedding: { type: [Number] }, generatedAt: { type: Date }, model: { type: String }, dimensions: { type: Number }, processingTime: { type: Number }, functionResults: [ { name: { type: String }, success: { type: Boolean }, result: { type: Schema.Types.Mixed }, error: { type: String }, executedAt: { type: Date } } ] }, default: void 0 } }); } schema.methods.getAIContent = function() { return this[config.field] || null; }; schema.methods.regenerateAI = async function() { try { this[config.field] = void 0; await processAI(this, config, provider); } catch (error) { throw new Error( `Failed to regenerate AI content: ${error instanceof Error ? error.message : "Unknown error"}` ); } }; if (config.model === "embedding") { schema.methods.calculateSimilarity = function(other) { const thisEmbedding = this[config.field]?.embedding; const otherEmbedding = other[config.field]?.embedding; if (!thisEmbedding || !otherEmbedding) { return 0; } return cosineSimilarity(thisEmbedding, otherEmbedding); }; schema.statics.semanticSearch = async function(query, options2 = {}) { const { limit = 10, threshold = 0.7, filter = {} } = options2; if (!query || typeof query !== "string") { throw new Error("Query string is required"); } try { const queryResult = await provider.generateEmbedding(query); const queryEmbedding = queryResult.embedding; const shouldUseVectorSearch = await determineSearchMethod( this, options2, vectorSearchConfig ); if (shouldUseVectorSearch) { await ensureVectorIndex( this, config.field, queryEmbedding.length, vectorSearchConfig ); return await performVectorSearch(this, queryEmbedding, config.field, { ...options2, indexName: vectorSearchConfig.indexName, numCandidates: options2.numCandidates || limit * 10 }); } else { return await performInMemorySearch( this, queryEmbedding, config.field, options2 ); } } catch (error) { throw new Error( `Semantic search failed: ${error instanceof Error ? error.message : "Unknown error"}` ); } }; schema.statics.findSimilar = async function(document, options2 = {}) { if (!document) { throw new Error("Reference document is required"); } const refEmbedding = document[config.field]?.embedding; if (!refEmbedding) { throw new Error("Document has no embedding"); } const { limit = 10, threshold = 0.7, filter = {} } = options2; try { const shouldUseVectorSearch = await determineSearchMethod( this, options2, vectorSearchConfig ); if (shouldUseVectorSearch) { await ensureVectorIndex( this, config.field, refEmbedding.length, vectorSearchConfig ); return await performVectorSearch(this, refEmbedding, config.field, { ...options2, filter: { ...filter, _id: { $ne: document._id } }, indexName: vectorSearchConfig.indexName, numCandidates: options2.numCandidates || limit * 10 }); } else { return await performInMemorySearch(this, refEmbedding, config.field, { ...options2, filter: { ...filter, _id: { $ne: document._id } } }); } } catch (error) { throw new Error( `Find similar failed: ${error instanceof Error ? error.message : "Unknown error"}` ); } }; } schema.pre("save", async function(next) { try { if (config.advanced?.skipOnUpdate && !this.isNew) { return next(); } if (this[config.field] && !config.advanced?.forceRegenerate) { return next(); } if (!hasContent(this, config)) { return next(); } await processAI(this, config, provider); next(); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown AI processing error"; if (config.advanced?.continueOnError !== false) { console.warn(`AI processing failed: ${errorMessage}`); next(); } else { next(new Error(`AI processing failed: ${errorMessage}`)); } } }); } async function determineSearchMethod(model, options, vectorSearchConfig) { if (vectorSearchConfig.enabled === false) { return false; } if (options.useVectorSearch !== void 0) { return options.useVectorSearch; } if (vectorSearchCache.has(model)) { return vectorSearchCache.get(model); } try { const isSupported = await detectVectorSearchSupport(model); vectorSearchCache.set(model, isSupported); return isSupported; } catch (error) { vectorSearchCache.set(model, false); return false; } } async function ensureVectorIndex(model, fieldName, dimensions, vectorSearchConfig) { try { await createVectorIndex(model, fieldName, dimensions, vectorSearchConfig); } catch (error) { console.warn( `Failed to create vector index: ${error instanceof Error ? error.message : "Unknown error"}` ); } } async function processAI(document, config, provider) { const docData = document.toObject(); delete docData._id; delete docData.__v; delete docData.createdAt; delete docData.updatedAt; delete docData[config.field]; let processedData = { ...docData }; if (config.includeFields && config.includeFields.length > 0) { processedData = {}; config.includeFields.forEach((field) => { if (docData[field] !== void 0) { processedData[field] = docData[field]; } }); } if (config.excludeFields && config.excludeFields.length > 0) { config.excludeFields.forEach((field) => { delete processedData[field]; }); } if (config.model === "summary") { const functionsWithDocument = config.functions?.map((func) => ({ ...func, handler: (args, _) => func.handler(args, document) })); const result = await provider.summarize( processedData, config.prompt, functionsWithDocument ); document[config.field] = result; } else if (config.model === "embedding") { const text = JSON.stringify(processedData); const result = await provider.generateEmbedding(text); document[config.field] = result; } } function hasContent(document, config) { const docObj = document.toObject(); delete docObj._id; delete docObj.__v; delete docObj.createdAt; delete docObj.updatedAt; delete docObj[config.field]; let processedData = { ...docObj }; if (config.includeFields && config.includeFields.length > 0) { processedData = {}; config.includeFields.forEach((field) => { if (docObj[field] !== void 0) { processedData[field] = docObj[field]; } }); } if (config.excludeFields && config.excludeFields.length > 0) { config.excludeFields.forEach((field) => { delete processedData[field]; }); } const content = JSON.stringify(processedData); return content.length > 20; } // src/types/functions.ts var QuickFunctions = { updateField: (fieldName, allowedValues) => { const valueParam = { type: "string", description: `New value for ${fieldName}`, required: true }; if (allowedValues && allowedValues.length > 0) { valueParam.enum = allowedValues; } return { name: `update_${fieldName}`, description: `Update the ${fieldName} field`, parameters: { value: valueParam }, handler: async (args, document) => { document[fieldName] = args.value; } }; }, scoreField: (fieldName, min = 0, max = 10) => ({ name: `score_${fieldName}`, description: `Score the ${fieldName} field between ${min} and ${max}`, parameters: { score: { type: "number", description: `Score for ${fieldName} (${min}-${max})`, required: true } }, handler: async (args, document) => { const clampedScore = Math.max(min, Math.min(max, args.score)); document[fieldName] = clampedScore; } }), manageTags: (fieldName = "tags") => ({ name: `manage_${fieldName}`, description: `Add or remove tags from ${fieldName} array`, parameters: { action: { type: "string", description: "Action to perform", enum: ["add", "remove", "replace"], required: true }, tags: { type: "array", description: "Tags to add, remove, or replace with", items: { type: "string", description: "Tag name" }, required: true } }, handler: async (args, document) => { const currentTags = document[fieldName] || []; switch (args.action) { case "add": document[fieldName] = [ .../* @__PURE__ */ new Set([...currentTags, ...args.tags]) ]; break; case "remove": document[fieldName] = currentTags.filter( (tag) => !args.tags.includes(tag) ); break; case "replace": document[fieldName] = args.tags; break; } } }) }; function createFunction(name, description, parameters, handler) { return { name, description, parameters, handler }; } // src/types/search.ts function isSearchResult(obj) { return obj && typeof obj.similarity === "number" && obj.document && typeof obj.metadata === "object"; } // src/types/models.ts function hasAIMethods(model) { return typeof model.semanticSearch === "function"; } function hasAIDocumentMe