UNPKG

one-search-mcp

Version:

One Search MCP Server, Web Search & Crawl & Scraper & Extract, support agent-browser, SearXNG, Tavily, DuckDuckGo, Bing, etc.

1 lines 95.1 kB
{"version":3,"sources":["../src/index.ts","../src/libs/logger/index.ts","../src/search/logger.ts","../src/search/bing.ts","../src/search/duckduckgo.ts","../src/search/searxng.ts","../src/search/tavily.ts","../src/libs/agent-browser/index.ts","../src/libs/browser/finder.ts","../src/libs/agent-browser/config.ts","../src/search/local.ts","../src/search/google.ts","../src/search/zhipu.ts","../src/search/exa.ts","../src/search/bocha.ts","../src/schemas.ts","../src/tools.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';\nimport { ISearchRequestOptions, ISearchResponse, SearchProvider } from './interface.js';\nimport { bingSearch, duckDuckGoSearch, searxngSearch, tavilySearch, localSearch, googleSearch, zhipuSearch, exaSearch, bochaSearch } from './search/index.js';\nimport { SEARCH_TOOL, EXTRACT_TOOL, SCRAPE_TOOL, MAP_TOOL } from './tools.js';\nimport type { SearchInput, MapInput, ScrapeInput, ExtractInput } from './schemas.js';\nimport { AgentBrowser } from './libs/agent-browser/index.js';\nimport dotenvx from '@dotenvx/dotenvx';\nimport { SafeSearchType } from 'duck-duck-scrape';\n\n// Load environment variables silently (suppress all output)\ndotenvx.config({ quiet: true });\n\n// search api\nconst SEARCH_API_URL = process.env.SEARCH_API_URL;\nconst SEARCH_API_KEY = process.env.SEARCH_API_KEY;\nconst SEARCH_PROVIDER: SearchProvider = process.env.SEARCH_PROVIDER as SearchProvider ?? 'local';\n\n// search query params\nconst SAFE_SEARCH = process.env.SAFE_SEARCH ?? 0;\nconst LIMIT = process.env.LIMIT ?? 10;\nconst CATEGORIES = process.env.CATEGORIES ?? 'general';\nconst ENGINES = process.env.ENGINES ?? 'all';\nconst FORMAT = process.env.FORMAT ?? 'json';\nconst LANGUAGE = process.env.LANGUAGE ?? 'auto';\nconst TIME_RANGE = process.env.TIME_RANGE ?? '';\nconst DEFAULT_TIMEOUT = process.env.TIMEOUT ?? 10000;\n\n// Server implementation using MCP SDK v1.25+ pattern\nconst server = new McpServer(\n {\n name: 'one-search-mcp',\n version: '1.1.2',\n },\n {\n capabilities: {\n tools: {},\n logging: {},\n },\n },\n);\n\nconst searchDefaultConfig = {\n limit: Number(LIMIT),\n categories: CATEGORIES,\n format: FORMAT,\n safesearch: SAFE_SEARCH,\n language: LANGUAGE,\n engines: ENGINES,\n time_range: TIME_RANGE,\n timeout: DEFAULT_TIMEOUT,\n};\n\n// Helper function to wrap tool handlers with logging and error handling\nfunction createToolHandler<TInput, TOutput>(\n toolName: string,\n handler: (args: TInput) => Promise<TOutput>,\n) {\n return async (args: TInput) => {\n const startTime = Date.now();\n\n try {\n await server.sendLoggingMessage({\n level: 'info',\n data: `[${new Date().toISOString()}] Request started for tool: [${toolName}]`,\n });\n\n const result = await handler(args);\n\n await server.sendLoggingMessage({\n level: 'info',\n data: `[${new Date().toISOString()}] Request completed in ${Date.now() - startTime}ms`,\n });\n\n return result;\n } catch (error) {\n await server.sendLoggingMessage({\n level: 'error',\n data: `[${new Date().toISOString()}] Error in ${toolName}: ${error}`,\n });\n const msg = error instanceof Error ? error.message : String(error);\n return {\n content: [\n {\n type: 'text' as const,\n text: msg,\n },\n ],\n isError: true,\n };\n }\n };\n}\n\n// Register tools using the new registerTool API\nserver.registerTool(\n SEARCH_TOOL.name,\n {\n description: SEARCH_TOOL.description,\n inputSchema: SEARCH_TOOL.schema,\n },\n createToolHandler(SEARCH_TOOL.name, async (args: SearchInput) => {\n const { results, success } = await processSearch({\n ...args,\n apiKey: SEARCH_API_KEY ?? '',\n apiUrl: SEARCH_API_URL,\n });\n\n if (!success) {\n throw new Error('Failed to search');\n }\n\n const resultsText = results.map((result) => (\n `Title: ${result.title}\nURL: ${result.url}\nDescription: ${result.snippet}\n${result.markdown ? `Content: ${result.markdown}` : ''}`\n ));\n\n return {\n content: [\n {\n type: 'text' as const,\n text: resultsText.join('\\n\\n'),\n },\n ],\n };\n }),\n);\n\nserver.registerTool(\n SCRAPE_TOOL.name,\n {\n description: SCRAPE_TOOL.description,\n inputSchema: SCRAPE_TOOL.schema,\n },\n createToolHandler(SCRAPE_TOOL.name, async (args: ScrapeInput) => {\n const { url, ...scrapeArgs } = args;\n const { content } = await processScrape(url, scrapeArgs);\n\n return {\n content,\n };\n }),\n);\n\nserver.registerTool(\n MAP_TOOL.name,\n {\n description: MAP_TOOL.description,\n inputSchema: MAP_TOOL.schema,\n },\n createToolHandler(MAP_TOOL.name, async (args: MapInput) => {\n const { content } = await processMapUrl(args.url, args);\n\n return {\n content,\n };\n }),\n);\n\nserver.registerTool(\n EXTRACT_TOOL.name,\n {\n description: EXTRACT_TOOL.description,\n inputSchema: EXTRACT_TOOL.schema,\n },\n createToolHandler(EXTRACT_TOOL.name, async (args: ExtractInput) => {\n const { content } = await processExtract(args);\n\n return {\n content,\n };\n }),\n);\n\n// Business logic functions\nasync function processSearch(args: ISearchRequestOptions): Promise<ISearchResponse> {\n switch (SEARCH_PROVIDER) {\n case 'searxng': {\n // merge default config with args\n const params = {\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n };\n\n // but categories and language have higher priority (ENV > args).\n const { categories, language } = searchDefaultConfig;\n\n if (categories) {\n params.categories = categories;\n }\n if (language) {\n params.language = language;\n }\n return await searxngSearch(params);\n }\n case 'tavily': {\n return await tavilySearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n });\n }\n case 'bing': {\n return await bingSearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n });\n }\n case 'duckduckgo': {\n const safeSearch = args.safeSearch ?? 0;\n const safeSearchOptions = [SafeSearchType.STRICT, SafeSearchType.MODERATE, SafeSearchType.OFF];\n return await duckDuckGoSearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n safeSearch: safeSearchOptions[safeSearch],\n });\n }\n case 'local': {\n return await localSearch({\n ...searchDefaultConfig,\n ...args,\n });\n }\n case 'google': {\n return await googleSearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n apiUrl: SEARCH_API_URL,\n });\n }\n case 'zhipu': {\n return await zhipuSearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n });\n }\n case 'exa': {\n return await exaSearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n });\n }\n case 'bocha': {\n return await bochaSearch({\n ...searchDefaultConfig,\n ...args,\n apiKey: SEARCH_API_KEY,\n });\n }\n default:\n throw new Error(`Unsupported search provider: ${SEARCH_PROVIDER}`);\n }\n}\n\nasync function processScrape(url: string, args: Omit<ScrapeInput, 'url'>): Promise<{\n content: Array<{ type: 'text'; text: string }>;\n result: any;\n success: boolean;\n}> {\n const browser = new AgentBrowser({\n headless: true,\n timeout: 30000,\n });\n\n try {\n const res = await browser.scrapeUrl(url, args);\n\n if (!res.success) {\n throw new Error(`Failed to scrape: ${res.error}`);\n }\n\n const content: string[] = [];\n\n if (res.markdown) {\n content.push(res.markdown);\n }\n\n if (res.rawHtml) {\n content.push(res.rawHtml);\n }\n\n if (res.links) {\n content.push(res.links.join('\\n'));\n }\n\n if (res.screenshot) {\n content.push(res.screenshot);\n }\n\n if (res.html) {\n content.push(res.html);\n }\n\n return {\n content: [\n {\n type: 'text' as const,\n text: content.join('\\n\\n') || 'No content found',\n },\n ],\n result: res,\n success: true,\n };\n } finally {\n await browser.close();\n }\n}\n\nasync function processMapUrl(url: string, args: Omit<MapInput, 'url'>): Promise<{\n content: Array<{ type: 'text'; text: string }>;\n result: string[];\n success: boolean;\n}> {\n const browser = new AgentBrowser({\n headless: true,\n timeout: 30000,\n });\n\n try {\n const res = await browser.mapUrl(url, args);\n\n if (!res.success) {\n throw new Error(`Failed to map: ${res.error}`);\n }\n\n if (!res.links) {\n throw new Error(`No links found from: ${url}`);\n }\n\n return {\n content: [\n {\n type: 'text' as const,\n text: res.links.join('\\n').trim(),\n },\n ],\n result: res.links,\n success: true,\n };\n } finally {\n await browser.close();\n }\n}\n\nasync function processExtract(args: ExtractInput): Promise<{\n content: Array<{ type: 'text'; text: string }>;\n}> {\n const { urls, prompt, systemPrompt, schema } = args;\n const browser = new AgentBrowser({\n headless: true,\n timeout: 30000,\n });\n\n try {\n const results: string[] = [];\n\n // Extract content from each URL\n for (const url of urls) {\n try {\n const res = await browser.scrapeUrl(url, {\n formats: ['markdown'],\n onlyMainContent: true,\n });\n\n if (res.success && res.markdown) {\n results.push(`## Content from ${url}\\n\\n${res.markdown}`);\n } else {\n results.push(`## Failed to extract from ${url}\\n\\nError: ${res.error || 'Unknown error'}`);\n }\n } catch (error) {\n const errorMsg = error instanceof Error ? error.message : String(error);\n results.push(`## Failed to extract from ${url}\\n\\nError: ${errorMsg}`);\n }\n }\n\n let finalText = results.join('\\n\\n---\\n\\n');\n\n // Add extraction instructions if provided\n if (prompt || systemPrompt || schema) {\n const instructions: string[] = [];\n\n if (systemPrompt) {\n instructions.push(`System Instructions: ${systemPrompt}`);\n }\n\n if (prompt) {\n instructions.push(`Extraction Task: ${prompt}`);\n }\n\n if (schema) {\n instructions.push(`Expected Schema:\\n${JSON.stringify(schema, null, 2)}`);\n }\n\n finalText = `${instructions.join('\\n\\n')}\\n\\n---\\n\\nExtracted Content:\\n\\n${finalText}`;\n }\n\n return {\n content: [\n {\n type: 'text' as const,\n text: finalText,\n },\n ],\n };\n } finally {\n await browser.close();\n }\n}\n\nasync function runServer(): Promise<void> {\n try {\n // Do NOT write to stdout before connecting - it will break MCP protocol\n const transport = new StdioServerTransport();\n await server.connect(transport);\n\n // Now we can send logging messages through MCP protocol\n await server.sendLoggingMessage({\n level: 'info',\n data: 'OneSearch MCP server started',\n });\n\n } catch (error) {\n const msg = error instanceof Error ? error.message : String(error);\n process.stderr.write(`Error starting server: ${msg}\\n`);\n process.exit(1);\n }\n}\n\n// run server\nrunServer().catch((error) => {\n const msg = error instanceof Error ? error.message : String(error);\n process.stderr.write(`Error running server: ${msg}\\n`);\n process.exit(1);\n});\n\n// export types\nexport * from './interface.js';\n","import pino from 'pino';\n\nexport interface Logger {\n info(message: string, ...args: any[]): void;\n error(message: string, ...args: any[]): void;\n success(message: string, ...args: any[]): void;\n warn(message: string, ...args: any[]): void;\n log(message: string, ...args: any[]): void;\n}\n\nexport class PinoLogger implements Logger {\n private logger: pino.Logger;\n\n constructor(name?: string) {\n this.logger = pino(\n {\n name: name || 'one-search-mcp',\n level: process.env.LOG_LEVEL || 'info',\n transport: process.env.NODE_ENV === 'development' ? {\n target: 'pino-pretty',\n options: {\n colorize: true,\n translateTime: 'SYS:standard',\n ignore: 'pid,hostname',\n },\n } : undefined,\n },\n // CRITICAL: Write to stderr, not stdout (MCP protocol requirement)\n process.stderr,\n );\n }\n\n private logWithLevel(level: 'info' | 'error' | 'warn', message: string, ...args: any[]): void {\n if (args.length > 0) {\n this.logger[level]({ data: args }, message);\n } else {\n this.logger[level](message);\n }\n }\n\n info(message: string, ...args: any[]): void {\n this.logWithLevel('info', message, ...args);\n }\n\n error(message: string, ...args: any[]): void {\n this.logWithLevel('error', message, ...args);\n }\n\n success(message: string, ...args: any[]): void {\n if (args.length > 0) {\n this.logger.info({ level: 'success', data: args }, message);\n } else {\n this.logger.info({ level: 'success' }, message);\n }\n }\n\n warn(message: string, ...args: any[]): void {\n this.logWithLevel('warn', message, ...args);\n }\n\n log(message: string, ...args: any[]): void {\n this.logWithLevel('info', message, ...args);\n }\n}\n\nexport const defaultLogger = new PinoLogger();\n","/**\n * Shared logger for search providers\n */\n\nimport { PinoLogger } from '../libs/logger/index.js';\n\nexport const searchLogger = new PinoLogger('search');\n","/**\n * Bing Search API\n * @reference https://learn.microsoft.com/en-us/previous-versions/bing/search-apis/bing-web-search/create-bing-search-service-resource\n */\nimport { ISearchRequestOptions, ISearchResponse } from '../interface.js';\nimport { searchLogger } from './logger.js';\n\nconst DEFAULT_TIMEOUT = 20000;\n\n\n/**\n * Options for performing a Bing search\n */\nexport interface BingSearchOptions {\n /**\n * Search query string\n */\n q: string;\n\n /**\n * Number of results to return\n */\n count?: number;\n\n /**\n * Result offset for pagination\n */\n offset?: number;\n\n /**\n * Market code (e.g., 'en-US')\n */\n mkt?: string;\n\n /**\n * Safe search filtering level\n */\n safeSearch?: 'Off' | 'Moderate' | 'Strict';\n\n /**\n * Bing API key\n */\n apiKey: string;\n\n /**\n * Bing Search API URL\n */\n apiUrl?: string;\n\n /**\n * Additional parameters supported by Bing Search API\n */\n [key: string]: any;\n}\n\n/**\n * Represents a web page result from Bing Search\n */\nexport interface BingSearchWebPage {\n /**\n * Title of the web page\n */\n name: string;\n\n /**\n * URL of the web page\n */\n url: string;\n\n /**\n * Text snippet from the web page\n */\n snippet: string;\n\n /**\n * Date the page was last crawled by Bing\n */\n dateLastCrawled?: string;\n\n /**\n * Display URL for the web page\n */\n displayUrl?: string;\n\n /**\n * Unique identifier for the result\n */\n id?: string;\n\n /**\n * Indicates if the content is family friendly\n */\n isFamilyFriendly?: boolean;\n\n /**\n * Indicates if the result is navigational\n */\n isNavigational?: boolean;\n\n /**\n * Language of the web page\n */\n language?: string;\n\n /**\n * Indicates if caching should be disabled\n */\n noCache?: boolean;\n\n /**\n * Name of the website\n */\n siteName?: string;\n\n /**\n * URL to a thumbnail image\n */\n thumbnailUrl?: string;\n}\n\n/**\n * Represents an image result from Bing Search\n */\nexport interface BingSearchImage {\n contentSize: string;\n contentUrl: string;\n datePublished: string;\n encodingFormat: string;\n height: number;\n width: number;\n hostPageDisplayUrl: string;\n hostPageUrl: string;\n name: string;\n thumbnail: {\n height: number;\n width: number;\n };\n thumbnailUrl: string;\n webSearchUrl: string;\n}\n\n/**\n * Represents a video result from Bing Search\n */\nexport interface BingSearchVideo {\n allowHttpsEmbed: boolean;\n allowMobileEmbed: boolean;\n contentUrl: string;\n creator?: {\n name: string;\n };\n datePublished: string;\n description: string;\n duration: string;\n embedHtml: string;\n encodingFormat: string;\n height: number;\n width: number;\n hostPageDisplayUrl: string;\n hostPageUrl: string;\n name: string;\n publisher?: {\n name: string;\n }[];\n thumbnail: {\n height: number;\n width: number;\n };\n thumbnailUrl: string;\n viewCount?: number;\n webSearchUrl: string;\n}\n\nexport interface BingSearchResponse {\n _type?: string;\n queryContext?: {\n originalQuery: string;\n };\n webPages?: {\n value: BingSearchWebPage[];\n totalEstimatedMatches?: number;\n someResultsRemoved?: boolean;\n webSearchUrl?: string;\n };\n images?: {\n value: BingSearchImage[];\n isFamilyFriendly?: boolean;\n readLink?: string;\n webSearchUrl?: string;\n id?: string;\n };\n videos?: {\n value: BingSearchVideo[];\n isFamilyFriendly?: boolean;\n readLink?: string;\n webSearchUrl?: string;\n id?: string;\n scenario?: string;\n };\n rankingResponse?: {\n mainline?: {\n items: {\n answerType: string;\n resultIndex?: number;\n value: {\n id: string;\n };\n }[];\n };\n };\n [key: string]: any; // Allow other response fields\n}\n\nexport async function bingSearch(options: ISearchRequestOptions): Promise<ISearchResponse> {\n const { query, limit = 10, safeSearch = 0, page = 1, apiUrl = 'https://api.bing.microsoft.com/v7.0/search', apiKey, language } = options;\n\n if (!query?.trim()) {\n throw new Error('Query cannot be empty');\n }\n\n if (!apiKey) {\n throw new Error('Bing API key is required');\n }\n\n const bingSafeSearchOptions = ['Off', 'Moderate', 'Strict'];\n\n const searchOptions = {\n q: query,\n count: limit,\n offset: (page - 1) * limit,\n mkt: language,\n safeSearch: bingSafeSearchOptions[safeSearch] as 'Off' | 'Moderate' | 'Strict',\n };\n\n const controller = new AbortController();\n const timeoutId = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT);\n\n try {\n const queryParams = new URLSearchParams();\n Object.entries(searchOptions).forEach(([key, value]) => {\n if (value !== undefined) {\n queryParams.set(key, value.toString());\n }\n });\n\n const res = await fetch(`${apiUrl}?${queryParams}`, {\n method: 'GET',\n headers: {\n 'Content-Type': 'application/json',\n 'Ocp-Apim-Subscription-Key': apiKey,\n },\n signal: controller.signal,\n });\n\n clearTimeout(timeoutId);\n\n if (!res.ok) {\n throw new Error(`Bing search error: ${res.status} ${res.statusText}`);\n }\n\n const data = await res.json();\n const serp = data.webPages?.value as Array<BingSearchWebPage>;\n const results = serp?.map((item: BingSearchWebPage) => ({\n title: item.name,\n snippet: item.snippet,\n url: item.url,\n source: item.siteName,\n thumbnailUrl: item.thumbnailUrl,\n language: item.language,\n image: null,\n video: null,\n engine: 'bing',\n })) ?? [];\n\n return {\n results,\n success: true,\n };\n } catch (err: unknown) {\n clearTimeout(timeoutId);\n const msg = err instanceof Error ? err.message : 'Bing search error.';\n searchLogger.error(msg);\n throw err;\n }\n}","import * as DDG from 'duck-duck-scrape';\nimport asyncRetry from 'async-retry';\nimport type { SearchOptions } from 'duck-duck-scrape';\nimport { ISearchRequestOptions, ISearchResponse } from '../interface.js';\nimport { searchLogger } from './logger.js';\n\nexport async function duckDuckGoSearch(options: Omit<ISearchRequestOptions, 'safeSearch'> & SearchOptions): Promise<ISearchResponse> {\n const { query, timeout = 10000, safeSearch = DDG.SafeSearchType.OFF, retry = { retries: 3 }, ...searchOptions } = options;\n\n if (!query?.trim()) {\n throw new Error('Query cannot be empty');\n }\n\n try {\n const res = await asyncRetry(\n () => {\n return DDG.search(query, {\n ...searchOptions,\n safeSearch,\n }, {\n // needle options\n response_timeout: timeout,\n });\n },\n retry,\n );\n\n const results = res ? {\n noResults: res.noResults,\n vqd: res.vqd,\n results: res.results,\n } : {\n noResults: true,\n vqd: '',\n results: [],\n };\n\n return {\n results: results.results.map((result) => ({\n title: result.title,\n snippet: result.description,\n url: result.url,\n source: result.hostname,\n image: null,\n video: null,\n engine: 'duckduckgo',\n })),\n success: true,\n };\n } catch (error) {\n const msg = error instanceof Error ? error.message : 'DuckDuckGo search error.';\n searchLogger.error(msg);\n throw error;\n }\n}\n","import url from 'node:url';\nimport { ISearchRequestOptions, ISearchResponse, ISearchResponseResult } from '../interface.js';\nimport { searchLogger } from './logger.js';\n\n/**\n * SearxNG Search API\n * @reference https://docs.searxng.org/dev/search_api.html\n */\nexport async function searxngSearch(params: ISearchRequestOptions): Promise<ISearchResponse> {\n const {\n query,\n page = 1,\n limit = 10,\n categories = 'general',\n engines = 'all',\n safeSearch = 0,\n format = 'json',\n language = 'auto',\n timeRange = '',\n timeout = 10000,\n apiKey,\n apiUrl,\n } = params;\n\n if (!query?.trim()) {\n throw new Error('Query cannot be empty');\n }\n\n if (!apiUrl) {\n throw new Error('SearxNG API URL is required');\n }\n\n const controller = new AbortController();\n const timeoutId = setTimeout(() => controller.abort(), Number(timeout));\n\n try {\n const config = {\n q: query,\n pageno: page,\n categories,\n format,\n safesearch: safeSearch,\n language,\n engines,\n time_range: timeRange,\n };\n\n const endpoint = `${apiUrl}/search`;\n\n const queryParams = url.format({ query: config });\n\n const headers: HeadersInit = {\n 'Content-Type': 'application/json',\n };\n\n if (apiKey) {\n headers['Authorization'] = `Bearer ${apiKey}`;\n }\n\n const res = await fetch(`${endpoint}${queryParams}`, {\n method: 'POST',\n headers,\n signal: controller.signal,\n });\n\n clearTimeout(timeoutId);\n const response = await res.json();\n if (response.results) {\n const list = (response.results as Array<Record<string, any>>).slice(0, limit);\n const results: ISearchResponseResult[] = list.map((item: Record<string, any>) => {\n const image = item.img_src ? {\n thumbnail: item.thumbnail_src,\n src: item.img_src,\n } : null;\n const video = item.iframe_src ? {\n thumbnail: item.thumbnail_src,\n src: item.iframe_src,\n } : null;\n return {\n title: item.title,\n snippet: item.content,\n url: item.url,\n source: item.source,\n image,\n video,\n engine: item.engine,\n };\n });\n return {\n results,\n success: true,\n };\n }\n return {\n results: [],\n success: false,\n };\n } catch (err: unknown) {\n clearTimeout(timeoutId);\n const msg = err instanceof Error ? err.message : 'Searxng search error.';\n searchLogger.error(msg);\n throw err;\n }\n}\n","/**\n * Tavily Search API\n * @reference https://docs.tavily.com/documentation/quickstart\n */\nimport { tavily, TavilySearchOptions } from '@tavily/core';\nimport { ISearchRequestOptions, ISearchResponse } from '../interface.js';\nimport { searchLogger } from './logger.js';\n\nconst DEFAULT_TIMEOUT = 20000;\n\nexport async function tavilySearch(options: ISearchRequestOptions): Promise<ISearchResponse> {\n const {\n query,\n limit = 10,\n categories = 'general',\n timeRange,\n apiKey,\n } = options;\n\n if (!query?.trim()) {\n throw new Error('Query cannot be empty');\n }\n\n if (!apiKey) {\n throw new Error('Tavily API key is required');\n }\n\n let timeoutId: NodeJS.Timeout | undefined;\n\n try {\n const tvly = tavily({\n apiKey,\n });\n\n const params: TavilySearchOptions = {\n topic: categories as TavilySearchOptions['topic'],\n timeRange: timeRange as TavilySearchOptions['timeRange'],\n maxResults: limit,\n };\n\n const res = await Promise.race([\n tvly.search(query, params),\n new Promise<never>((_, reject) => {\n timeoutId = setTimeout(() => reject(new Error('Tavily search timeout')), DEFAULT_TIMEOUT);\n }),\n ]);\n\n clearTimeout(timeoutId);\n\n const results = res.results.map(item => ({\n title: item.title,\n url: item.url,\n snippet: item.content,\n engine: 'tavily',\n }));\n\n return {\n results,\n success: true,\n };\n } catch (error) {\n clearTimeout(timeoutId);\n const msg = error instanceof Error ? error.message : 'Tavily search error.';\n searchLogger.error(msg);\n throw error;\n }\n}\n","import { BrowserManager } from 'agent-browser/dist/browser.js';\nimport type { Page } from 'playwright-core';\nimport TurndownService from 'turndown';\nimport { gfm } from 'turndown-plugin-gfm';\nimport * as cheerio from 'cheerio';\nimport { BrowserFinder } from '../browser/finder.js';\nimport { defaultLogger } from '../logger/index.js';\nimport { BROWSER_CONFIG } from './config.js';\nimport type {\n AgentBrowserOptions,\n ScrapeResult,\n ScrapeOptions,\n MapResult,\n MapOptions,\n SearchResult,\n SearchEngine,\n SearchOptions,\n} from './types.js';\n\nexport class AgentBrowser {\n private browser: BrowserManager;\n private turndown: TurndownService;\n private browserPath?: string;\n\n constructor(private options: AgentBrowserOptions = {}) {\n this.browser = new BrowserManager();\n\n // Try to find local browser installation\n try {\n const finder = new BrowserFinder();\n const { executable } = finder.findBrowser();\n this.browserPath = executable;\n } catch (error) {\n // If no local browser found, agent-browser will try to use Playwright's Chromium\n this.browserPath = undefined;\n }\n\n // Initialize Turndown for HTML to Markdown conversion\n this.turndown = new TurndownService({\n headingStyle: 'atx',\n codeBlockStyle: 'fenced',\n });\n this.turndown.use(gfm);\n }\n\n /**\n * Ensure browser is launched\n */\n private async ensureLaunched(): Promise<void> {\n if (!this.browser.isLaunched()) {\n try {\n await this.browser.launch({\n id: `session-${Date.now()}`,\n action: 'launch',\n headless: this.options.headless ?? true,\n executablePath: this.browserPath,\n });\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n\n // Check if this is a browser not installed error\n if (errorMessage.includes('Executable') || errorMessage.includes('browser')) {\n throw new Error(\n 'Browser not found. Please install one of the following:\\n' +\n ' - Google Chrome: https://www.google.com/chrome/\\n' +\n ' - Microsoft Edge: https://www.microsoft.com/edge\\n' +\n ' - Chromium: https://www.chromium.org/getting-involved/download-chromium/\\n' +\n 'Or install via Playwright:\\n' +\n ' npx playwright install chromium\\n' +\n `Original error: ${errorMessage}`,\n );\n }\n\n throw error;\n }\n }\n }\n\n /**\n * Get current page\n */\n private async getPage(): Promise<Page> {\n await this.ensureLaunched();\n return this.browser.getPage();\n }\n\n /**\n * Navigate to URL\n */\n async navigate(url: string): Promise<void> {\n const page = await this.getPage();\n await page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.options.timeout });\n }\n\n /**\n * Get page HTML\n */\n async getHtml(): Promise<string> {\n const page = await this.getPage();\n return await page.content();\n }\n\n /**\n * Get page text\n */\n async getText(): Promise<string> {\n const page = await this.getPage();\n return await page.evaluate(() => document.body.innerText);\n }\n\n /**\n * Take screenshot\n */\n async screenshot(): Promise<string> {\n const page = await this.getPage();\n const screenshot = await page.screenshot({\n type: 'png',\n fullPage: false,\n });\n return `data:image/png;base64,${screenshot.toString('base64')}`;\n }\n\n /**\n * Wait for time\n */\n async wait(ms: number): Promise<void> {\n await new Promise((resolve) => setTimeout(resolve, ms));\n }\n\n /**\n * Close browser\n */\n async close(): Promise<void> {\n if (this.browser.isLaunched()) {\n await this.browser.close();\n }\n }\n\n /**\n * Scrape a URL and extract content\n */\n async scrapeUrl(url: string, options: ScrapeOptions = {}): Promise<ScrapeResult> {\n try {\n await this.navigate(url);\n\n // Wait for page load\n if (options.waitFor) {\n await this.wait(options.waitFor);\n } else {\n await this.wait(BROWSER_CONFIG.DEFAULT_WAIT_MS);\n }\n\n const result: ScrapeResult = { success: true };\n const formats = options.formats || ['markdown'];\n\n // Get HTML\n const html = await this.getHtml();\n\n if (formats.includes('markdown')) {\n result.markdown = this.turndown.turndown(html);\n }\n\n if (formats.includes('html') || formats.includes('rawHtml')) {\n result.html = html;\n result.rawHtml = html;\n }\n\n if (formats.includes('links')) {\n result.links = this.extractLinks(html, url);\n }\n\n if (formats.includes('screenshot') || formats.includes('screenshot@fullPage')) {\n result.screenshot = await this.screenshot();\n }\n\n return result;\n } catch (error) {\n return {\n success: false,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n }\n\n /**\n * Map URLs from a starting URL\n */\n async mapUrl(url: string, options: MapOptions = {}): Promise<MapResult> {\n try {\n await this.navigate(url);\n await this.wait(BROWSER_CONFIG.DEFAULT_WAIT_MS);\n\n const html = await this.getHtml();\n let links = this.extractLinks(html, url);\n\n // Filter links\n if (options.search) {\n links = links.filter((link) =>\n link.toLowerCase().includes(options.search!.toLowerCase()),\n );\n }\n\n if (!options.includeSubdomains) {\n const baseHost = new URL(url).hostname;\n links = links.filter((link) => {\n try {\n return new URL(link).hostname === baseHost;\n } catch {\n return false;\n }\n });\n }\n\n if (options.limit) {\n links = links.slice(0, options.limit);\n }\n\n return {\n success: true,\n links,\n };\n } catch (error) {\n return {\n success: false,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n }\n\n /**\n * Search using a search engine\n */\n async search(options: SearchOptions): Promise<SearchResult[]> {\n const { query, engine, limit = 10 } = options;\n const searchUrl = this.getSearchUrl(engine, query);\n\n await this.navigate(searchUrl);\n await this.wait(BROWSER_CONFIG.SEARCH_WAIT_MS);\n\n const html = await this.getHtml();\n const results = this.extractSearchResults(engine, html);\n\n return results.slice(0, limit);\n }\n\n /**\n * Get search URL for different engines\n */\n private getSearchUrl(engine: SearchEngine, query: string): string {\n const encodedQuery = encodeURIComponent(query);\n switch (engine) {\n case 'google':\n return `https://www.google.com/search?q=${encodedQuery}`;\n case 'bing':\n return `https://www.bing.com/search?q=${encodedQuery}`;\n case 'baidu':\n return `https://www.baidu.com/s?wd=${encodedQuery}`;\n case 'sogou':\n return `https://www.sogou.com/web?query=${encodedQuery}`;\n default:\n throw new Error(`Unsupported search engine: ${engine}`);\n }\n }\n\n /**\n * Extract search results from HTML\n */\n private extractSearchResults(engine: SearchEngine, html: string): SearchResult[] {\n try {\n switch (engine) {\n case 'google':\n return this.extractGoogleResults(html);\n case 'bing':\n return this.extractBingResults(html);\n case 'baidu':\n return this.extractBaiduResults(html);\n case 'sogou':\n return this.extractSogouResults(html);\n default:\n return [];\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n defaultLogger.warn(`Failed to extract ${engine} search results: ${errorMessage}`);\n return [];\n }\n }\n\n /**\n * Extract Google search results using cheerio\n */\n private extractGoogleResults(html: string): SearchResult[] {\n const $ = cheerio.load(html);\n const results: SearchResult[] = [];\n\n // Google uses div.g for search results\n $('div.g').each((_, element) => {\n const $el = $(element);\n const $link = $el.find('a[href]').first();\n const $title = $el.find('h3').first();\n const $snippet = $el.find('div.VwiC3b, div[data-sncf]').first();\n\n const url = $link.attr('href');\n const title = $title.text().trim();\n const snippet = $snippet.text().trim();\n\n if (url && title && !url.startsWith('/search')) {\n results.push({ title, url, snippet });\n }\n });\n\n return results;\n }\n\n /**\n * Extract Bing search results using cheerio\n */\n private extractBingResults(html: string): SearchResult[] {\n const $ = cheerio.load(html);\n const results: SearchResult[] = [];\n\n // Bing uses li.b_algo for search results\n $('li.b_algo').each((_, element) => {\n const $el = $(element);\n const $link = $el.find('h2 a').first();\n const $snippet = $el.find('p, div.b_caption p').first();\n\n const url = $link.attr('href');\n const title = $link.text().trim();\n const snippet = $snippet.text().trim();\n\n if (url && title) {\n results.push({ title, url, snippet });\n }\n });\n\n return results;\n }\n\n /**\n * Extract Baidu search results using cheerio\n */\n private extractBaiduResults(html: string): SearchResult[] {\n const $ = cheerio.load(html);\n const results: SearchResult[] = [];\n\n // Baidu uses div.result for search results\n $('div.result').each((_, element) => {\n const $el = $(element);\n const $link = $el.find('h3 a').first();\n const $snippet = $el.find('div.c-abstract, div[class*=\"abstract\"]').first();\n\n const url = $link.attr('href');\n const title = $link.text().trim();\n const snippet = $snippet.text().trim();\n\n if (url && title) {\n results.push({ title, url, snippet });\n }\n });\n\n return results;\n }\n\n /**\n * Extract Sogou search results using cheerio\n */\n private extractSogouResults(html: string): SearchResult[] {\n const $ = cheerio.load(html);\n const results: SearchResult[] = [];\n\n // Sogou uses div.vrwrap for search results\n $('div.vrwrap').each((_, element) => {\n const $el = $(element);\n const $link = $el.find('h3 a').first();\n const $snippet = $el.find('p.str-text, p[class*=\"text\"]').first();\n\n const url = $link.attr('href');\n const title = $link.text().trim();\n const snippet = $snippet.text().trim();\n\n if (url && title) {\n results.push({ title, url, snippet });\n }\n });\n\n return results;\n }\n\n /**\n * Extract links from HTML\n */\n private extractLinks(html: string, baseUrl: string): string[] {\n const linkRegex = /<a[^>]+href=[\"']([^\"']+)[\"']/gi;\n const links: string[] = [];\n let match;\n\n while ((match = linkRegex.exec(html)) !== null) {\n try {\n const link = new URL(match[1], baseUrl).href;\n if (!links.includes(link)) {\n links.push(link);\n }\n } catch {\n // Ignore invalid links\n }\n }\n\n return links;\n }\n\n}\n\nexport * from './types.js';\nexport * from './config.js';\n","/**\n * The following code is modified based on\n * https://github.com/egoist/local-web-search/blob/main/src/find-browser.ts\n * Copy from\n * https://github.com/bytedance/UI-TARS-desktop/blob/main/packages/agent-infra/browser/src/browser-finder.ts\n *\n * MIT Licensed\n * Copyright (c) 2025 ChatWise (https://chatwise.app) <kevin@chatwise.app>\n * https://github.com/egoist/local-web-search/blob/main/LICENSE\n */\n\nimport * as fs from 'fs';\nimport * as path from 'path';\nimport * as os from 'os';\nimport { Logger, defaultLogger } from '../logger/index.js';\n\n/**\n * Interface defining browser locations and configurations\n * Contains paths and settings for different operating systems\n * @interface Browser\n */\ninterface Browser {\n /**\n * Browser name identifier\n */\n name: string;\n\n /**\n * Executable paths by platform\n * @property {string} win32 - Windows executable path\n * @property {string} darwin - macOS executable path\n * @property {string} linux - Linux executable path\n */\n executable: {\n win32: string;\n darwin: string;\n linux: string;\n };\n\n /**\n * User data directory paths by platform\n * @property {string} win32 - Windows user data directory\n * @property {string} darwin - macOS user data directory\n * @property {string} linux - Linux user data directory\n */\n userDataDir: {\n win32: string;\n darwin: string;\n linux: string;\n };\n}\n\n/**\n * Class responsible for finding and managing browser installations\n * Detects installed browsers and their profiles across different platforms\n */\nexport class BrowserFinder {\n /**\n * Logger instance for diagnostic output\n */\n private logger: Logger;\n\n /**\n * Creates a new BrowserFinder instance\n * @param {Logger} [logger] - Optional custom logger\n */\n constructor(logger?: Logger) {\n this.logger = logger ?? defaultLogger;\n }\n\n /**\n * Getter that returns the list of supported browsers with their platform-specific paths\n * @returns {Browser[]} Array of browser configurations\n * @private\n */\n private get browsers(): Browser[] {\n // Get HOME_DIR inside the getter to ensure it's always current\n const HOME_DIR = os.homedir();\n const LOCAL_APP_DATA = process.env.LOCALAPPDATA;\n\n return [\n {\n name: 'Chromium',\n executable: {\n win32: 'C:\\\\Program Files\\\\Chromium\\\\Application\\\\chrome.exe',\n darwin: '/Applications/Chromium.app/Contents/MacOS/Chromium',\n linux: '/usr/bin/chromium',\n },\n userDataDir: {\n win32: `${LOCAL_APP_DATA}\\\\Chromium\\\\User Data`,\n darwin: `${HOME_DIR}/Library/Application Support/Chromium`,\n linux: `${HOME_DIR}/.config/chromium`,\n },\n },\n {\n name: 'Google Chrome',\n executable: {\n win32: 'C:\\\\Program Files\\\\Google\\\\Chrome\\\\Application\\\\chrome.exe',\n darwin:\n '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',\n linux: '/usr/bin/google-chrome',\n },\n userDataDir: {\n win32: `${LOCAL_APP_DATA}\\\\Google\\\\Chrome\\\\User Data`,\n darwin: `${HOME_DIR}/Library/Application Support/Google/Chrome`,\n linux: `${HOME_DIR}/.config/google-chrome`,\n },\n },\n {\n name: 'Google Chrome Canary',\n executable: {\n win32:\n 'C:\\\\Program Files\\\\Google\\\\Chrome Canary\\\\Application\\\\chrome.exe',\n darwin:\n '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',\n linux: '/usr/bin/google-chrome-canary',\n },\n userDataDir: {\n win32: `${LOCAL_APP_DATA}\\\\Google\\\\Chrome Canary\\\\User Data`,\n darwin: `${HOME_DIR}/Library/Application Support/Google/Chrome Canary`,\n linux: `${HOME_DIR}/.config/google-chrome-canary`,\n },\n },\n {\n name: 'Microsoft Edge',\n executable: {\n win32: 'C:\\\\Program Files (x86)\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe',\n darwin: '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',\n linux: '/usr/bin/microsoft-edge',\n },\n userDataDir: {\n win32: `${LOCAL_APP_DATA}\\\\Microsoft\\\\Edge\\\\User Data`,\n darwin: `${HOME_DIR}/Library/Application Support/Microsoft Edge`,\n linux: `${HOME_DIR}/.config/microsoft-edge`,\n },\n },\n ];\n }\n\n /**\n * Find a specific browser or the first available browser\n * @param {string} [name] - Optional browser name to find\n * @returns {{ executable: string; userDataDir: string }} Browser executable and user data paths\n * @throws {Error} If no supported browser is found or the platform is unsupported\n */\n findBrowser(name?: string): {\n executable: string;\n userDataDir: string;\n } {\n const platform = process.platform;\n this.logger.info('Finding browser on platform:', platform);\n\n if (platform !== 'darwin' && platform !== 'win32' && platform !== 'linux') {\n const error = new Error(`Unsupported platform: ${platform}`);\n this.logger.error(error.message);\n throw error;\n }\n\n const browser = name\n ? this.browsers.find(\n (b) => b.name === name && fs.existsSync(b.executable[platform]),\n )\n : this.browsers.find((b) => fs.existsSync(b.executable[platform]));\n\n this.logger.log('browser', browser);\n\n if (!browser) {\n const error = name\n ? new Error(`Cannot find browser: ${name}`)\n : new Error(\n 'Cannot find a supported browser on your system. Please install Chrome, Chromium, Edge, or Brave.',\n );\n this.logger.error(error.message);\n throw error;\n }\n\n const result = {\n executable: browser.executable[platform],\n userDataDir: browser.userDataDir[platform],\n };\n\n this.logger.success(`Found browser: ${browser.name}`);\n this.logger.info('Browser details:', result);\n\n return result;\n }\n\n /**\n * Get browser profiles for a specific browser\n * Reads the Local State file to extract profile information\n * @param {string} [browserName] - Optional browser name to get profiles for\n * @returns {Array<{ displayName: string; path: string }>} Array of profile objects with display names and paths\n */\n getBrowserProfiles(\n browserName?: string,\n ): Array<{ displayName: string; path: string }> {\n const browser = this.findBrowser(browserName);\n\n try {\n const localState = JSON.parse(\n fs.readFileSync(path.join(browser.userDataDir, 'Local State'), 'utf8'),\n );\n const profileInfo = localState.profile.info_cache;\n\n return Object.entries(profileInfo).map(\n ([profileName, info]: [string, any]) => ({\n displayName: info.name,\n path: path.join(browser.userDataDir, profileName),\n }),\n );\n } catch (error) {\n return [];\n }\n }\n\n /**\n * Legacy method for backwards compatibility\n * Finds Chrome browser executable path\n * @deprecated Use findBrowser instead\n * @returns {string | null} Chrome executable path or null if not found\n */\n findChrome(): string | null {\n try {\n const { executable } = this.findBrowser('Google Chrome');\n return executable;\n } catch {\n return null;\n }\n }\n}\n","/**\n * Configuration constants for agent-browser\n */\nexport const BROWSER_CONFIG = {\n /** Default wait time after page load (ms) */\n DEFAULT_WAIT_MS: 2000,\n\n /** Wait time for search results to load (ms) */\n SEARCH_WAIT_MS: 3000,\n\n /** Default browser timeout (ms) */\n DEFAULT_TIMEOUT: 30000,\n\n /** Run browser in headless mode by default */\n DEFAULT_HEADLESS: true,\n} as const;\n\n/**\n * Valid search engines\n */\nexport const VALID_SEARCH_ENGINES = ['bing', 'google', 'baidu', 'sogou'] as const;\n","import { ISearchRequestOptions, ISearchResponse, ISearchResponseResult } from '../interface.js';\nimport { AgentBrowser, type SearchEngine, VALID_SEARCH_ENGINES } from '../libs/agent-browser/index.js';\nimport { PinoLogger } from '../libs/logger/index.js';\n\nconst logger = new PinoLogger('[LocalSearch]');\n\n/**\n * Validate if a string is a valid search engine\n */\nfunction isValidEngine(engine: string): engine is SearchEngine {\n return VALID_SEARCH_ENGINES.includes(engine as SearchEngine);\n}\n\n/**\n * Convert SearchResult to ISearchResponseResult\n */\nfunction toSearchResponseResult(result: { title: string; url: string; snippet: string; content?: string }, engine: string): ISearchResponseResult {\n return {\n title: result.title,\n snippet: result.snippet,\n url: result.url,\n markdown: result.content,\n engine,\n };\n}\n\nexport async function localSearch(options: ISearchRequestOptions): Promise<ISearchResponse> {\n const { query, limit = 10 } = options;\n let { engines = 'all' } = options;\n\n if (engines === 'all') {\n engines = 'bing,google,baidu,sogou';\n }\n\n const engineList = engines.split(',').map(e => e.trim()).filter(Boolean);\n\n if (engineList.length === 0) {\n throw new Error('engines is required');\n }\n\n // Validate engines\n const validEngines = engineList.filter(isValidEngine);\n const invalidEngines = engineList.filter(e => !isValidEngine(e));\n\n if (invalidEngines.length > 0) {\n logger.warn(`Invalid search engines ignored: ${invalidEngines.join(', ')}`);\n }\n\n if (validEngines.length === 0) {\n throw new Error(`No valid search engines provided. Valid engines: ${VALID_SEARCH_ENGINES.join(', ')}`);\n }\n\n const browser = new AgentBrowser({\n headless: true,\n timeout: 30000,\n });\n\n try {\n const results: ISearchResponseResult[] = [];\n\n for (const engine of validEngines) {\n try {\n logger.info(`Searching with engine: ${engine}`);\n const searchResults = await browser.search({\n query,\n engine,\n limit,\n });\n\n if (searchResults.length > 0) {\n // Convert SearchResult to ISearchResponseResult\n const converted = searchResults.map(r => toSearchResponseResult(r, engine));\n results.push(...converted);\n logger.info(`Found ${searchResults.length} results from ${engine}`);\n break; // Use first successful engine\n }\n } catch (err) {\n logger.error(`Failed to search with ${engine}:`, err);\n // Continue to next engine\n }\n }\n\n logger.info(`Total results found: ${results.length}`);\n\n return {\n results,\n success: true,\n };\n } catch (err: unknown) {\n const msg = err instanceof Error ? err.message : 'Local search error.';\n logger.error(msg, err);\n throw err;\n } finally {\n await browser.close();\n }\n}\n","/**\n * Google Custom Search API\n * @reference https://developers.google.com/custom-search/v1/overview\n */\n\nimport { ISearchRequestOptions, ISearchResponse, ISearchResponseResult } from '../interface.js';\nimport { searchLogger } from './logger.js';\n\nconst GOOGLE_SEARCH_ENDPOINT = 'https://www.googleapis.com/customsearch/v1';\nconst DEFAULT_TIMEOUT = 20000;\n\ninterface GoogleSearchItem {\n title: string;\n link: string;\n snippet: string;\n displayLink?: string;\n formattedUrl?: string;\n pagemap?: {\n cse_thumbnail?: Array<{ src: string }>;\n };\n}\n\ninterface GoogleSearchResponse {\n items?: GoogleSearchItem[];\n error?: {\n message: string;\n };\n}\n\nexport async function googleSearch(options: ISearchRequestOptions): Promise<ISearchResponse> {\n const { query, apiKey, apiUrl, limit = 10, language } = options;\n\n if (!query?.trim()) {\n throw new Error('Query cannot be empty');\n }\n\n if (!apiKey || !apiUrl) {\n throw new Error('Google search requires SEARCH_API_KEY and SEARCH_API_URL (Search Engine ID)');\n }\n\n const params = new URLSearchParams({\n key: apiKey,\n cx: apiUrl,\n q: query,\n num: String(Math.min(limit, 10)),\n });\n\n if (language && language !== 'auto') {\n params.set('lr', `lang_${language}`);\n }\n\n const controller = new AbortController();\n const timeoutId = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT);\n\n try {\n const response = await fetch(`${GOOGLE_SEARCH_ENDPOINT}?${params}`,