UNPKG

@web3ai/cli

Version:

Your AI-powered command-line companion for seamless Web3 development. Ask questions, get code suggestions, and accelerate your blockchain projects.

839 lines (828 loc) 29.4 kB
// src/services/vector-db/vector-db.ts import { OpenAIEmbeddings } from "@langchain/openai"; import { MemoryVectorStore } from "langchain/vectorstores/memory"; import { Document } from "@langchain/core/documents"; import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; import axios from "axios"; import * as cheerio from "cheerio"; import * as fs2 from "fs"; import * as path2 from "path"; // src/services/config/config.ts import JoyCon from "joycon"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import toml from "smol-toml"; import { z } from "zod"; var configDirPath = path.join(os.homedir(), ".config", "web3cli"); var AICommandVariableSchema = z.union([ z.string().describe("a shell command to run"), z.object({ type: z.literal("input"), message: z.string() }).describe("get text input from the user"), z.object({ type: z.literal("select"), message: z.string(), choices: z.array( z.object({ value: z.string(), title: z.string() }) ) }).describe("get a choice from the user") ]); var AICommandSchema = z.object({ command: z.string().describe("the cli command"), example: z.string().optional().describe("example to show in cli help"), description: z.string().optional().describe("description to show in cli help"), variables: z.record(AICommandVariableSchema).optional(), prompt: z.string().describe("the prompt to send to the model"), require_stdin: z.boolean().optional().describe("Require piping output from another program to Web3CLI") }); var ConfigSchema = z.object({ default_model: z.string().optional(), openai_api_key: z.string().optional().describe('Default to the "OPENAI_API_KEY" environment variable'), openai_api_url: z.string().optional().describe('Default to the "OPENAI_API_URL" environment variable'), anthropic_api_key: z.string().optional().describe('Default to the "ANTHROPIC_API_KEY" environment variable'), gemini_api_key: z.string().optional().describe('Default to the "GEMINI_API_KEY" environment variable'), gemini_api_url: z.string().optional().describe('Default to the "GEMINI_API_URL" environment variable'), groq_api_key: z.string().optional().describe('Default to the "GROQ_API_KEY" environment variable'), groq_api_url: z.string().optional().describe('Default to the "GROQ_API_URL" environment variable'), mistral_api_key: z.string().optional().describe('Default to the "MISTRAL_API_KEY" environment variable'), mistral_api_url: z.string().optional().describe('Default to the "MISTRAL_API_URL" environment variable'), etherscan_api_key: z.string().optional().describe('Default to the "ETHERSCAN_API_KEY" environment variable'), ollama_host: z.string().optional().describe('Default to the "OLLAMA_HOST" environment variable'), commands: z.array(AICommandSchema).optional() }); function loadConfig() { const joycon = new JoyCon.default(); joycon.addLoader({ test: /\.toml$/, loadSync: (filepath) => { const content = fs.readFileSync(filepath, "utf-8"); return toml.parse(content); } }); function safeLoad(filenames, cwd, stopDir) { try { const result = joycon.loadSync(filenames, cwd, stopDir); return result.data; } catch (err) { const message = err instanceof Error ? err.message : String(err); console.warn( `Warning: ignored malformed config while reading ${filenames.join(", ")} \u2014 ${message}` ); return void 0; } } const globalConfig = safeLoad( ["web3cli.json", "web3cli.toml"], configDirPath, path.dirname(configDirPath) ); const localConfig = safeLoad( ["web3cli.json", "web3cli.toml"], process.cwd(), path.dirname(process.cwd()) ); let baseConfig = void 0; let commandsFromConfig = []; if (globalConfig) { baseConfig = { ...globalConfig }; commandsFromConfig = [...globalConfig.commands || []]; } else if (localConfig) { baseConfig = { ...localConfig }; commandsFromConfig = [...localConfig.commands || []]; } else { baseConfig = {}; } const config = { ...baseConfig, // Spread, ensuring it's treated as Config commands: commandsFromConfig }; const envVarMapping = { openai_api_key: "OPENAI_API_KEY", openai_api_url: "OPENAI_API_URL", anthropic_api_key: "ANTHROPIC_API_KEY", gemini_api_key: "GEMINI_API_KEY", gemini_api_url: "GEMINI_API_URL", groq_api_key: "GROQ_API_KEY", groq_api_url: "GROQ_API_URL", mistral_api_key: "MISTRAL_API_KEY", mistral_api_url: "MISTRAL_API_URL", etherscan_api_key: "ETHERSCAN_API_KEY", ollama_host: "OLLAMA_HOST" }; for (const [configKey, envVar] of Object.entries(envVarMapping)) { if (process.env[envVar] && !config[configKey]) { config[configKey] = process.env[envVar]; } } return config; } // src/services/vector-db/vector-db.ts import { Readability } from "@mozilla/readability"; import { JSDOM } from "jsdom"; import { parse as parseHTML } from "node-html-parser"; var VectorDB = class { embeddings; collections; textSplitter; dataDir; constructor(options = {}) { const config = loadConfig(); this.embeddings = new OpenAIEmbeddings({ openAIApiKey: config.openai_api_key, modelName: "text-embedding-ada-002", // Use older model for better compatibility batchSize: 512, // Process embeddings in smaller batches stripNewLines: true // Remove newlines to avoid errors }); this.textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: options.chunkSize || 500, // Smaller chunks for better embedding chunkOverlap: options.chunkOverlap || 100 }); this.dataDir = path2.join(process.cwd(), ".vector-db"); if (!fs2.existsSync(this.dataDir)) { fs2.mkdirSync(this.dataDir, { recursive: true }); } this.collections = options.collections || {}; this.loadCollections(); } /** * Load all collections from disk */ async loadCollections() { try { if (!fs2.existsSync(this.dataDir)) { fs2.mkdirSync(this.dataDir, { recursive: true }); return; } const collectionsPath = path2.join(this.dataDir, "collections.json"); if (!fs2.existsSync(collectionsPath)) { return; } const collectionsData = JSON.parse(fs2.readFileSync(collectionsPath, "utf-8")); for (const [name, data] of Object.entries(collectionsData)) { console.log(`Loading collection ${name} from ${data}`); const collectionPath = path2.join(this.dataDir, `${name}.json`); if (fs2.existsSync(collectionPath)) { try { const jsonData = fs2.readFileSync(collectionPath, "utf-8"); const documentData = JSON.parse(jsonData); if (!Array.isArray(documentData)) { console.warn(`Invalid document data in collection ${name}, expected array`); continue; } const validDocs = documentData.filter((doc) => { return doc && (doc.pageContent !== void 0 || doc.metadata && Object.keys(doc.metadata).length > 0); }).map((doc) => { return new Document({ pageContent: doc.pageContent && doc.pageContent.trim() !== "" ? doc.pageContent : "This document contains no extractable text content.", metadata: doc.metadata || {} }); }); if (validDocs.length > 0) { this.collections[name] = await MemoryVectorStore.fromDocuments(validDocs, this.embeddings); } } catch (error) { console.error(`Error loading collection ${name}:`, error); } } } } catch (error) { console.error("Error loading collections:", error); } } /** * Save collections metadata to disk */ async saveCollectionsMetadata() { try { let existingData = {}; const collectionsPath = path2.join(this.dataDir, "collections.json"); if (fs2.existsSync(collectionsPath)) { try { existingData = JSON.parse(fs2.readFileSync(collectionsPath, "utf-8")); } catch (readError) { console.error("Error reading existing collections metadata:", readError); } } const collectionsData = Object.keys(this.collections).reduce((acc, name) => { acc[name] = { name, timestamp: (/* @__PURE__ */ new Date()).toISOString(), updated: true }; return acc; }, existingData); if (!fs2.existsSync(this.dataDir)) { fs2.mkdirSync(this.dataDir, { recursive: true }); } fs2.writeFileSync(collectionsPath, JSON.stringify(collectionsData, null, 2)); } catch (error) { console.error("Error saving collections metadata:", error); } } /** * Save a specific collection to disk */ async saveCollection(name) { const collection = this.collections[name]; if (!collection) return; try { const vectorsData = collection.memoryVectors; if (!vectorsData || !Array.isArray(vectorsData) || vectorsData.length === 0) { return; } const documentsToSave = vectorsData.map((item) => { const pageContent = item.pageContent && item.pageContent.trim() !== "" ? item.pageContent : "This document contains no extractable text content."; return { pageContent, metadata: item.metadata || {} }; }); if (!fs2.existsSync(this.dataDir)) { fs2.mkdirSync(this.dataDir, { recursive: true }); } const collectionPath = path2.join(this.dataDir, `${name}.json`); fs2.writeFileSync(collectionPath, JSON.stringify(documentsToSave, null, 2)); await this.saveCollectionsMetadata(); } catch (error) { console.error(`Error saving collection ${name}:`, error); } } /** * Get a collection, creating it if it doesn't exist */ async getCollection(name) { if (this.collections[name]) { return this.collections[name]; } const collectionPath = path2.join(this.dataDir, `${name}.json`); if (fs2.existsSync(collectionPath)) { try { const jsonData = fs2.readFileSync(collectionPath, "utf-8"); const documentData = JSON.parse(jsonData); if (!Array.isArray(documentData)) { console.warn(`Invalid document data in collection ${name}, expected array`); } else { const validDocs = documentData.filter((doc) => { return doc && (doc.pageContent !== void 0 || doc.metadata && Object.keys(doc.metadata).length > 0); }).map((doc) => { return new Document({ pageContent: doc.pageContent || "No content", metadata: doc.metadata || {} }); }); if (validDocs.length > 0) { this.collections[name] = await MemoryVectorStore.fromDocuments( validDocs, this.embeddings ); await this.saveCollectionsMetadata(); return this.collections[name]; } } } catch (error) { console.error(`Error loading collection ${name}:`, error); } } this.collections[name] = new MemoryVectorStore(this.embeddings); await this.saveCollectionsMetadata(); await this.saveCollection(name); return this.collections[name]; } /** * Extract content from HTML using Mozilla's Readability * This provides high-quality content extraction similar to browser reader modes */ extractContentWithReadability(html, url) { try { const dom = new JSDOM(html, { url }); try { const head = dom.window.document.querySelector("head"); if (head) { const base = dom.window.document.createElement("base"); base.href = url; head.insertBefore(base, head.firstChild); } } catch (error) { console.error("Error adding base element:", error); } const reader = new Readability(dom.window.document, { // Configure Readability options for better extraction keepClasses: true, charThreshold: 100, // More lenient threshold classesToPreserve: ["content", "article", "doc", "documentation", "post", "text", "body"] }); const article = reader.parse(); if (article && (article.textContent || article.content)) { let fullTextContent = article.textContent || ""; if (!fullTextContent || fullTextContent.trim().length < 100) { if (article.content) { try { fullTextContent = this.extractTextFromHTML(article.content); } catch (parseError) { console.error("Error parsing article content:", parseError); } } } return { title: article.title || "", content: article.content || "", textContent: fullTextContent || article.textContent || article.content || "", excerpt: article.excerpt || "", siteName: article.siteName || "", author: article.byline || "", success: true }; } return this.extractContentWithHTMLParser(html, url); } catch (error) { console.error("Error extracting content with Readability:", error); return this.extractContentWithHTMLParser(html, url); } } /** * Extract content using node-html-parser as a fallback */ extractContentWithHTMLParser(html, url) { try { const root = parseHTML(html); const titleElement = root.querySelector("title"); const h1Element = root.querySelector("h1"); const title = titleElement ? titleElement.text : h1Element ? h1Element.text : new URL(url).pathname.split("/").pop() || url; const $ = cheerio.load(html); $("script, style, nav, footer, header, .sidebar, .navigation, .menu, .ads, .banner, .cookie, .popup").remove(); const mainSelectors = "main, article, .content, .documentation, .docs, #content, #main, .article, .post, .entry, section"; let mainContent = $(mainSelectors); if (mainContent.length === 0) { mainContent = $("body"); } let textContent = ""; mainContent.find("h1, h2, h3, h4, h5, h6").each((i, el) => { const text = $(el).text().trim(); if (text) { const level = parseInt(el.tagName.substring(1).toLowerCase()); textContent += ` ${"#".repeat(level)} ${text} `; } }); mainContent.find('p, div > text, .text, [class*="text"], [class*="content"]').each((i, el) => { if ($(el).parents("h1, h2, h3, h4, h5, h6, li").length === 0) { const text = $(el).text().trim(); if (text) { textContent += `${text} `; } } }); mainContent.find("li").each((i, el) => { const text = $(el).text().trim(); if (text) { textContent += `\u2022 ${text} `; } }); if (textContent.trim().length < 100) { textContent = mainContent.text().trim(); } const cleanText = textContent.replace(/\s+/g, " ").trim(); return { title, content: $.html(mainContent) || html, textContent: cleanText || "No content could be extracted from this page.", success: cleanText.length > 0 }; } catch (error) { console.error("Error extracting with HTML parser:", error); return { title: new URL(url).pathname.split("/").pop() || url, content: "", textContent: "Error extracting content from page.", success: false }; } } /** * Extract headings from an HTML element */ // private extractHeadings(element: any): string { // let result = ''; // try { // const headings = element.querySelectorAll('h1, h2, h3, h4, h5, h6'); // for (const heading of headings) { // const text = heading.text.trim(); // if (text) { // const tagName = heading.tagName.toLowerCase(); // const level = parseInt(tagName.substring(1)); // const prefix = '#'.repeat(level) + ' '; // result += `\n\n${prefix}${text}\n\n`; // } // } // } catch (error) { // console.error("Error extracting headings:", error); // } // return result; // } /** * Extract paragraphs from an HTML element */ // private extractParagraphs(element: any): string { // let result = ''; // console.log(element); // try { // const paragraphs = element.querySelectorAll('p, div > text, .text, [class*="text"], [class*="content"]'); // for (const p of paragraphs) { // const text = p.text.trim(); // if (text) { // result += `${text}\n\n`; // } // } // } catch (error) { // console.error("Error extracting paragraphs:", error); // } // return result; // } /** * Extract lists from an HTML element */ // private extractLists(element: any): string { // let result = ''; // try { // const lists = element.querySelectorAll('ul, ol'); // for (const list of lists) { // const items = list.querySelectorAll('li'); // for (const item of items) { // const text = item.text.trim(); // if (text) { // result += `• ${text}\n`; // } // } // result += '\n'; // } // } catch (error) { // console.error("Error extracting lists:", error); // } // return result; // } /** * Extract text content from HTML (enhanced method using cheerio) */ extractTextFromHTML(html) { try { const $ = cheerio.load(html); $("script, style, nav, footer, header, .sidebar, .menu, .ads, .banner, .cookie, .popup").remove(); const mainContentSelectors = [ "main", "article", ".content", ".documentation", ".docs", "#content", "#main", ".document", ".rst-content", ".body", ".post", ".entry", "section", "[role=main]", ".main", ".article" ]; let mainContent = null; for (const selector of mainContentSelectors) { const element = $(selector); if (element.length > 0) { mainContent = element; break; } } if (!mainContent || mainContent.length === 0) { mainContent = $("body"); } let extractedText = ""; mainContent.find("h1, h2, h3, h4, h5, h6").each((i, el) => { const text = $(el).text().trim(); if (text.length > 0) { const tagName = el.tagName.toLowerCase(); const level = parseInt(tagName.substring(1)); const prefix = "#".repeat(level) + " "; extractedText += ` ${prefix}${text} `; } }); mainContent.find("p").each((i, el) => { const text = $(el).text().trim(); if (text.length > 0) { extractedText += `${text} `; } }); mainContent.find("div").each((i, el) => { if ($(el).find("p, h1, h2, h3, h4, h5, h6, ul, ol").length === 0) { const text = $(el).text().trim(); if (text.length > 0) { extractedText += `${text} `; } } }); mainContent.find("ul, ol").each((i, listEl) => { $(listEl).find("li").each((j, liEl) => { const text = $(liEl).text().trim(); if (text.length > 0) { extractedText += `\u2022 ${text} `; } }); extractedText += "\n"; }); mainContent.find("table").each((i, tableEl) => { extractedText += "\nTable content:\n"; $(tableEl).find("tr").each((j, trEl) => { const rowText = []; $(trEl).find("td, th").each((k, cellEl) => { rowText.push($(cellEl).text().trim()); }); if (rowText.length > 0) { extractedText += rowText.join(" | ") + "\n"; } }); extractedText += "\n"; }); mainContent.find("pre, code").each((i, el) => { const text = $(el).text().trim(); if (text.length > 0) { extractedText += ` \`\`\` ${text} \`\`\` `; } }); if (extractedText.trim().length < 100) { extractedText = mainContent.text().trim(); } const cleanedText = extractedText.replace(/\n{3,}/g, "\n\n").replace(/\s+/g, " ").trim(); return cleanedText.length > 0 ? cleanedText : "No content could be extracted from this page."; } catch (error) { console.error("Error extracting text from HTML:", error); return "Error extracting content from page."; } } /** * Extract title from HTML */ // private extractTitleFromHTML(html: string, fallbackUrl: string): string { // try { // const $ = cheerio.load(html); // // Try to get the title // const title = $("title").text().trim(); // if (title) return title; // // Try to get an h1 // const h1 = $("h1").first().text().trim(); // if (h1) return h1; // // Use the URL's last segment as fallback // return fallbackUrl.split("/").pop() || fallbackUrl; // } catch (error) { // return fallbackUrl.split("/").pop() || fallbackUrl; // } // } /** * Extract links from HTML that are on the same domain */ extractLinks(html, baseUrl) { try { const $ = cheerio.load(html); const links = []; const baseUrlObj = new URL(baseUrl); $("a").each((_, element) => { const href = $(element).attr("href"); if (!href) return; try { let fullUrl; if (href.startsWith("http")) { fullUrl = new URL(href); } else { fullUrl = new URL(href, baseUrl); } if (fullUrl.hostname === baseUrlObj.hostname) { links.push(fullUrl.href); } } catch (error) { } }); return [...new Set(links)]; } catch (error) { console.error("Error extracting links:", error); return []; } } /** * Add documents from a URL to the vector database */ async addDocs(collectionName, url, options = {}) { try { const collection = await this.getCollection(collectionName); const processedUrls = /* @__PURE__ */ new Set(); const urlQueue = [{ url, depth: 0 }]; let addedChunks = 0; const maxPages = Math.min(options.maxPages || 30, 30); const maxDepth = options.maxDepth || 3; while (urlQueue.length > 0 && processedUrls.size < maxPages) { const { url: currentUrl, depth } = urlQueue.shift(); if (processedUrls.has(currentUrl)) continue; processedUrls.add(currentUrl); console.log(`Fetching ${currentUrl} (depth: ${depth})`); try { const response = await axios.get(currentUrl); const html = response.data; const extractedContent = this.extractContentWithReadability(html, currentUrl); if (!extractedContent.success || !extractedContent.textContent || extractedContent.textContent.trim().length === 0) { continue; } const docs = await this.textSplitter.createDocuments([extractedContent.textContent], [{ source: currentUrl, title: extractedContent.title, url: currentUrl, siteName: extractedContent.siteName, author: extractedContent.author, crawlTime: (/* @__PURE__ */ new Date()).toISOString() }]); if (docs.length > 0) { const validDocs = docs.map((doc) => { if (!doc.pageContent || doc.pageContent.trim().length === 0) { doc.pageContent = "This document contains no extractable text content."; } return doc; }); await collection.addDocuments(validDocs); addedChunks += validDocs.length; } if (options.crawl && depth < maxDepth) { const links = this.extractLinks(html, currentUrl); for (const link of links) { if (!processedUrls.has(link)) { urlQueue.push({ url: link, depth: depth + 1 }); } } } } catch (error) { console.error(`Error processing ${currentUrl}:`, error); } await new Promise((resolve) => setTimeout(resolve, 500)); } if (addedChunks > 0) { await this.saveCollection(collectionName); } return addedChunks; } catch (error) { console.error(`Error adding documents to collection ${collectionName}:`, error); return 0; } } /** * Add a file to the vector database */ async addFile(collectionName, filePath, metadata = {}) { const collection = await this.getCollection(collectionName); const text = fs2.readFileSync(filePath, "utf-8"); if (!text || text.trim().length === 0) { return 0; } const docs = await this.textSplitter.createDocuments([text], [{ source: filePath, ...metadata }]); if (docs.length > 0) { await collection.addDocuments(docs); await this.saveCollection(collectionName); return docs.length; } return 0; } /** * Search for similar documents in a collection */ async search(collectionName, query, k = 5) { try { if (!query || query.trim().length === 0) { throw new Error("Search query cannot be empty"); } const collection = await this.getCollection(collectionName); const vectorsData = collection.memoryVectors; if (!vectorsData || !Array.isArray(vectorsData) || vectorsData.length === 0) { return []; } const results = await collection.similaritySearch(query, k); const validResults = results.map((doc) => { const newDoc = new Document({ pageContent: doc.pageContent && doc.pageContent.trim() !== "" ? doc.pageContent : "This document contains no extractable text content.", metadata: { ...doc.metadata, title: doc.metadata?.title || doc.metadata?.source?.split("/").pop() || "Untitled" } }); return newDoc; }); return validResults; } catch (error) { console.error(`Error searching collection ${collectionName}:`, error); return []; } } /** * List all available collections */ async listCollections() { try { const collectionsPath = path2.join(this.dataDir, "collections.json"); let collectionsFromMetadata = []; let collectionsFromFiles = []; if (fs2.existsSync(collectionsPath)) { try { const collectionsData = JSON.parse(fs2.readFileSync(collectionsPath, "utf-8")); collectionsFromMetadata = Object.keys(collectionsData); } catch (error) { console.error("Error parsing collections metadata:", error); } } if (fs2.existsSync(this.dataDir)) { try { const files = fs2.readdirSync(this.dataDir); collectionsFromFiles = files.filter((file) => file.endsWith(".json") && file !== "collections.json").map((file) => file.replace(".json", "")); } catch (error) { console.error("Error scanning collections directory:", error); } } const allCollections = [.../* @__PURE__ */ new Set([...collectionsFromMetadata, ...collectionsFromFiles])]; const validCollections = allCollections.filter((name) => { const collectionPath = path2.join(this.dataDir, `${name}.json`); return fs2.existsSync(collectionPath); }); if (validCollections.length > collectionsFromMetadata.length) { const newCollections = validCollections.filter((name) => !collectionsFromMetadata.includes(name)); if (newCollections.length > 0) { for (const name of newCollections) { await this.getCollection(name); } } } return validCollections; } catch (error) { console.error("Error listing collections:", error); return []; } } /** * Add text directly to the vector database */ async addText(collectionName, text, metadata = {}) { try { if (!text || text.trim().length === 0) { return 0; } const collection = await this.getCollection(collectionName); const docs = await this.textSplitter.createDocuments([text], [{ ...metadata }]); const validDocs = docs.map((doc) => { if (!doc.pageContent || doc.pageContent === "No content") { doc.pageContent = text.substring(0, Math.min(text.length, 500)); } return doc; }); if (validDocs.length > 0) { await collection.addDocuments(validDocs); await this.saveCollection(collectionName); return validDocs.length; } return 0; } catch (error) { console.error(`Error adding text to collection ${collectionName}:`, error); return 0; } } /** * Perform similarity search (alias to search method) */ async similaritySearch(collectionName, query, k = 5) { return this.search(collectionName, query, k); } }; export { configDirPath, loadConfig, VectorDB };