mcp-omnisearch
Version:
MCP server for integrating Omnisearch with LLMs
3,000 lines • 115 kB
JavaScript
#!/usr/bin/env node
import { StdioTransport } from "@tmcp/transport-stdio";
import { readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { ValibotJsonSchemaAdapter } from "@tmcp/adapter-valibot";
import { McpServer } from "tmcp";
import * as v from "valibot";
import { randomUUID } from "node:crypto";
import { tmpdir } from "node:os";
import { Octokit } from "octokit";
//#region src/config/env.ts
const TAVILY_API_KEY = process.env.TAVILY_API_KEY;
const BRAVE_API_KEY = process.env.BRAVE_API_KEY;
const KAGI_API_KEY = process.env.KAGI_API_KEY;
const GITHUB_API_KEY = process.env.GITHUB_API_KEY;
const EXA_API_KEY = process.env.EXA_API_KEY;
const LINKUP_API_KEY = process.env.LINKUP_API_KEY;
const FIRECRAWL_API_KEY = process.env.FIRECRAWL_API_KEY;
const FIRECRAWL_BASE_URL = process.env.FIRECRAWL_BASE_URL;
const config = {
search: {
tavily: {
api_key: TAVILY_API_KEY,
base_url: "https://api.tavily.com",
timeout: 3e4
},
brave: {
api_key: BRAVE_API_KEY,
base_url: "https://api.search.brave.com/res/v1",
timeout: 1e4
},
kagi: {
api_key: KAGI_API_KEY,
base_url: "https://kagi.com/api/v0",
timeout: 2e4
},
github: {
api_key: GITHUB_API_KEY,
base_url: "https://api.github.com",
timeout: 2e4
},
exa: {
api_key: EXA_API_KEY,
base_url: "https://api.exa.ai",
timeout: 3e4
}
},
ai_response: {
kagi_fastgpt: {
api_key: KAGI_API_KEY,
base_url: "https://kagi.com/api/v0/fastgpt",
timeout: 3e4
},
exa_answer: {
api_key: EXA_API_KEY,
base_url: "https://api.exa.ai",
timeout: 3e4
},
linkup: {
api_key: LINKUP_API_KEY,
base_url: "https://api.linkup.so/v1",
timeout: 3e4
},
tavily_research: {
api_key: TAVILY_API_KEY,
base_url: "https://api.tavily.com",
request_timeout: 1e4
}
},
processing: {
kagi_summarizer: {
api_key: KAGI_API_KEY,
base_url: "https://kagi.com/api/v0/summarize",
timeout: 3e4
},
tavily_extract: {
api_key: TAVILY_API_KEY,
base_url: "https://api.tavily.com",
timeout: 3e4
},
tavily_crawl: {
api_key: TAVILY_API_KEY,
base_url: "https://api.tavily.com",
timeout: 15e4
},
tavily_map: {
api_key: TAVILY_API_KEY,
base_url: "https://api.tavily.com",
timeout: 15e4
},
firecrawl_scrape: {
api_key: FIRECRAWL_API_KEY,
base_url: FIRECRAWL_BASE_URL ? `${FIRECRAWL_BASE_URL}/v2/scrape` : "https://api.firecrawl.dev/v2/scrape",
timeout: 6e4
},
firecrawl_crawl: {
api_key: FIRECRAWL_API_KEY,
base_url: FIRECRAWL_BASE_URL ? `${FIRECRAWL_BASE_URL}/v2/crawl` : "https://api.firecrawl.dev/v2/crawl",
timeout: 12e4
},
firecrawl_map: {
api_key: FIRECRAWL_API_KEY,
base_url: FIRECRAWL_BASE_URL ? `${FIRECRAWL_BASE_URL}/v2/map` : "https://api.firecrawl.dev/v2/map",
timeout: 6e4
},
firecrawl_extract: {
api_key: FIRECRAWL_API_KEY,
base_url: FIRECRAWL_BASE_URL ? `${FIRECRAWL_BASE_URL}/v2/extract` : "https://api.firecrawl.dev/v2/extract",
timeout: 6e4
},
firecrawl_actions: {
api_key: FIRECRAWL_API_KEY,
base_url: FIRECRAWL_BASE_URL ? `${FIRECRAWL_BASE_URL}/v2/scrape` : "https://api.firecrawl.dev/v2/scrape",
timeout: 9e4
},
exa_contents: {
api_key: EXA_API_KEY,
base_url: "https://api.exa.ai",
timeout: 3e4
},
exa_similar: {
api_key: EXA_API_KEY,
base_url: "https://api.exa.ai",
timeout: 3e4
}
},
enhancement: { kagi_enrichment: {
api_key: KAGI_API_KEY,
base_url: "https://kagi.com/api/v0/enrich",
timeout: 2e4
} }
};
const remote_deployment_markers = [
"AWS_LAMBDA_FUNCTION_NAME",
"CONTAINER",
"DOCKER_CONTAINER",
"FLY_APP_NAME",
"K_SERVICE",
"RENDER",
"VERCEL"
];
const should_warn_for_local_file_offload = (env = process.env) => env.OMNISEARCH_LARGE_RESULT_MODE === "file" && remote_deployment_markers.some((marker) => Boolean(env[marker]));
const warn_for_local_file_offload = (env = process.env, warn = console.warn) => {
if (!should_warn_for_local_file_offload(env)) return;
warn("Warning: OMNISEARCH_LARGE_RESULT_MODE=file returns server-side temp-file paths and is only useful for local shared-filesystem stdio clients. Use OMNISEARCH_LARGE_RESULT_MODE=inline for remote, hosted, or containerized MCP deployments.");
};
const validate_config = () => {
const missing_keys = [];
const available_keys = [];
if (!TAVILY_API_KEY) missing_keys.push("TAVILY_API_KEY");
else available_keys.push("TAVILY_API_KEY");
if (!BRAVE_API_KEY) missing_keys.push("BRAVE_API_KEY");
else available_keys.push("BRAVE_API_KEY");
if (!KAGI_API_KEY) missing_keys.push("KAGI_API_KEY");
else available_keys.push("KAGI_API_KEY");
if (!GITHUB_API_KEY) missing_keys.push("GITHUB_API_KEY");
else available_keys.push("GITHUB_API_KEY");
if (!FIRECRAWL_API_KEY) missing_keys.push("FIRECRAWL_API_KEY");
else available_keys.push("FIRECRAWL_API_KEY");
if (!EXA_API_KEY) missing_keys.push("EXA_API_KEY");
else available_keys.push("EXA_API_KEY");
if (!LINKUP_API_KEY) missing_keys.push("LINKUP_API_KEY");
else available_keys.push("LINKUP_API_KEY");
if (available_keys.length > 0) console.error(`Found API keys for: ${available_keys.join(", ")}`);
else console.error("Warning: No API keys found. No providers will be available.");
if (missing_keys.length > 0) console.warn(`Missing API keys for: ${missing_keys.join(", ")}. Some providers will not be available.`);
warn_for_local_file_offload();
};
//#endregion
//#region src/common/types.ts
var ProviderError = class extends Error {
type;
provider;
details;
constructor(type, message, provider, details) {
super(message);
this.type = type;
this.provider = provider;
this.details = details;
this.name = "ProviderError";
}
};
//#endregion
//#region src/common/errors.ts
const provider_error = (type, message, provider, details = {}) => new ProviderError(type, message, provider, details);
const normalize_provider_http_error = (provider, status, message) => {
switch (status) {
case 400:
case 422: return provider_error("INVALID_INPUT", `Invalid request: ${message}`, provider, {
status,
retryable: false
});
case 401:
case 403: return provider_error("AUTH_ERROR", status === 401 ? "Invalid API key" : "API key does not have access to this endpoint", provider, {
status,
retryable: false
});
case 408: return provider_error("TIMEOUT", `${provider} API request timed out`, provider, {
status,
retryable: true
});
case 429: return provider_error("RATE_LIMIT", `Rate limit exceeded for ${provider}`, provider, {
status,
retryable: true
});
default:
if (status >= 500) return provider_error("TRANSIENT_PROVIDER_ERROR", `${provider} API internal error`, provider, {
status,
retryable: true
});
return provider_error("API_ERROR", `Unexpected error: ${message}`, provider, {
status,
retryable: false
});
}
};
function handle_provider_error(error, provider_name, operation = "operation") {
if (error instanceof ProviderError) throw error;
throw new ProviderError("API_ERROR", `Failed to ${operation}: ${error instanceof Error ? error.message : "Unknown error"}`, provider_name);
}
const sanitize_query = (query) => {
return query.trim().replace(/[\n\r]+/g, " ");
};
const create_error_response = (error) => {
if (error instanceof ProviderError) return {
error: error.message,
type: error.type,
provider: error.provider,
retryable: error.details?.retryable ?? false
};
return {
error: `Unexpected error: ${error.message}`,
type: "API_ERROR",
retryable: false
};
};
//#endregion
//#region src/common/http.ts
const tryParseJson = (text) => {
if (!text) return void 0;
try {
return JSON.parse(text);
} catch {
return;
}
};
const get_error_message = (body) => {
if (typeof body !== "object" || body === null) return void 0;
for (const key of [
"message",
"error",
"detail"
]) if (key in body) {
const value = body[key];
if (typeof value === "string") return value;
}
};
const http_json = async (provider, url, options = {}) => {
const res = await fetch(url, options);
const raw = await res.text();
const body = tryParseJson(raw);
if (!(res.ok || options.expectedStatuses && options.expectedStatuses.includes(res.status))) {
const message = get_error_message(body) || raw || res.statusText;
throw normalize_provider_http_error(provider, res.status, message);
}
return body ?? raw;
};
//#endregion
//#region src/common/provider-response.ts
const parse_provider_response = (provider, schema, data) => {
const result = v.safeParse(schema, data);
if (result.success) return result.output;
throw new ProviderError("PROVIDER_ERROR", `Malformed ${provider} response: ${v.summarize(result.issues)}`, provider, { issue_count: result.issues.length });
};
//#endregion
//#region src/common/retry.ts
const delay = (ms) => {
return new Promise((resolve) => setTimeout(resolve, ms));
};
const is_object_with_name = (error) => typeof error === "object" && error !== null && "name" in error && typeof error.name === "string";
const is_retryable_error = (error) => {
if (error instanceof ProviderError) {
if (typeof error.details?.retryable === "boolean") return error.details.retryable;
if (error.type === "RATE_LIMIT" || error.type === "TIMEOUT" || error.type === "TRANSIENT_PROVIDER_ERROR") return true;
const status = error.details?.status;
return status !== void 0 && (status === 408 || status === 429 || status >= 500);
}
if (is_object_with_name(error)) return error.name === "AbortError" || error.name === "TimeoutError" || error.name === "TypeError";
return false;
};
const normalize_retry_options = (max_retries_or_options = {}, initial_delay) => {
const options = typeof max_retries_or_options === "number" ? {
max_retries: max_retries_or_options,
initial_delay: initial_delay ?? 1e3
} : max_retries_or_options;
return {
max_retries: options.max_retries ?? 3,
initial_delay: options.initial_delay ?? 1e3,
jitter_ratio: options.jitter_ratio ?? .2,
random: options.random ?? Math.random,
should_retry: options.should_retry ?? is_retryable_error
};
};
const apply_jitter = (delay_time, jitter_ratio, random) => {
if (jitter_ratio <= 0) return delay_time;
const jitter = 1 + (random() * 2 - 1) * jitter_ratio;
return Math.max(0, Math.round(delay_time * jitter));
};
const retry_with_backoff = async (fn, max_retries_or_options, initial_delay) => {
const options = normalize_retry_options(max_retries_or_options, initial_delay);
let retries = 0;
while (true) try {
return await fn();
} catch (error) {
if (retries >= options.max_retries || !options.should_retry(error)) throw error;
const base_delay = options.initial_delay * Math.pow(2, retries);
const delay_time = apply_jitter(base_delay, options.jitter_ratio, options.random);
await delay(delay_time);
retries++;
}
};
//#endregion
//#region src/common/validation.ts
const normalize_api_key = (raw) => {
return raw.trim().replace(/^(['"])(.*)\1$/, "$2");
};
const validate_api_key = (key, provider) => {
if (!key) throw new ProviderError("INVALID_INPUT", `API key not found for ${provider}`, provider);
return normalize_api_key(key);
};
const is_api_key_valid = (key, provider) => {
if (!key || key.trim() === "") {
console.warn(`API key not found or empty for ${provider}`);
return false;
}
return true;
};
const is_valid_url = (url) => {
try {
new URL(url);
return true;
} catch {
return false;
}
};
const validate_processing_urls = (url, provider_name) => {
const urls = Array.isArray(url) ? url : [url];
for (const u of urls) if (!is_valid_url(u)) throw new ProviderError("INVALID_INPUT", `Invalid URL provided: ${u}`, provider_name);
return urls;
};
//#endregion
//#region src/providers/ai-response/exa-answer/index.ts
const exa_answer_response_schema = v.object({
answer: v.string(),
citations: v.optional(v.array(v.object({
id: v.string(),
title: v.string(),
url: v.string(),
publishedDate: v.optional(v.string()),
text: v.optional(v.string()),
image: v.optional(v.string()),
favicon: v.optional(v.string())
}))),
requestId: v.string()
});
var ExaAnswerProvider = class {
name = "exa_answer";
description = "Get direct AI-generated answers to questions using Exa Answer API";
async search(params) {
const api_key = validate_api_key(config.ai_response.exa_answer.api_key, this.name);
const search_request = async () => {
try {
const raw_data = await http_json(this.name, `${config.ai_response.exa_answer.base_url}/answer`, {
method: "POST",
headers: {
"x-api-key": api_key,
"Content-Type": "application/json"
},
body: JSON.stringify({ query: sanitize_query(params.query) }),
signal: AbortSignal.timeout(config.ai_response.exa_answer.timeout)
});
const data = parse_provider_response(this.name, exa_answer_response_schema, raw_data);
const results = [{
title: "AI Answer",
url: "",
snippet: data.answer,
score: 1,
source_provider: this.name,
metadata: {
requestId: data.requestId,
type: "ai_answer",
citations_count: data.citations?.length || 0
}
}];
if (data.citations && data.citations.length > 0) {
const limit = params.limit ?? data.citations.length;
const citation_results = data.citations.slice(0, limit).map((citation, index) => ({
title: citation.title,
url: citation.url,
snippet: citation.text || "Source reference",
score: .9 - index * .01,
source_provider: this.name,
metadata: {
id: citation.id,
publishedDate: citation.publishedDate,
type: "citation"
}
}));
results.push(...citation_results);
}
return results;
} catch (error) {
handle_provider_error(error, this.name, "fetch AI response");
}
};
return retry_with_backoff(search_request);
}
};
//#endregion
//#region src/providers/ai-response/kagi-fastgpt/index.ts
const kagi_fastgpt_response_schema = v.object({
meta: v.object({
id: v.string(),
node: v.string(),
ms: v.number()
}),
data: v.object({
output: v.string(),
tokens: v.number(),
references: v.array(v.object({
title: v.string(),
snippet: v.string(),
url: v.string()
}))
})
});
var KagiFastGPTProvider = class {
name = "kagi_fastgpt";
description = "Quick AI-generated answers with citations, optimized for rapid response (900ms typical start time). Runs full search underneath for enriched answers.";
async search(params) {
const response = await this.get_answer(params.query);
const results = [];
results.push({
title: "Kagi FastGPT Response",
url: "https://kagi.com/fastgpt",
snippet: response.data.output,
source_provider: this.name
});
if (response.data.references && response.data.references.length > 0) results.push(...response.data.references.map((ref) => ({
title: ref.title,
url: ref.url,
snippet: ref.snippet,
source_provider: this.name
})));
const filtered_results = results.filter((result) => result.title && result.url && result.snippet);
if (params.limit && params.limit > 0) return filtered_results.slice(0, params.limit);
return filtered_results;
}
async get_answer(query, options = {}) {
const api_key = validate_api_key(config.ai_response.kagi_fastgpt.api_key, this.name);
const final_options = {
cache: true,
web_search: true,
...options
};
try {
const raw_data = await http_json(this.name, "https://kagi.com/api/v0/fastgpt", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bot ${api_key}`
},
body: JSON.stringify({
query,
cache: final_options.cache,
web_search: final_options.web_search
}),
signal: AbortSignal.timeout(config.ai_response.kagi_fastgpt.timeout)
});
return parse_provider_response(this.name, kagi_fastgpt_response_schema, raw_data);
} catch (error) {
const error_message = error instanceof Error ? error.message : String(error);
throw new Error(`Failed to get Kagi FastGPT answer: ${error_message}`);
}
}
};
//#endregion
//#region src/providers/ai-response/linkup/index.ts
const linkup_sourced_answer_response_schema = v.object({
answer: v.string(),
sources: v.array(v.object({
favicon: v.string(),
name: v.string(),
snippet: v.string(),
url: v.string()
}))
});
var LinkupProvider = class {
name = "linkup";
description = "AI-powered deep search with sourced answers via Linkup. Uses agentic search with standard depth for balanced speed and accuracy.";
async search(params) {
const api_key = validate_api_key(config.ai_response.linkup.api_key, this.name);
const search_request = async () => {
try {
const request_body = {
q: sanitize_query(params.query),
depth: "standard",
outputType: "sourcedAnswer"
};
if (params.include_domains && params.include_domains.length > 0) request_body.includeDomains = params.include_domains;
if (params.exclude_domains && params.exclude_domains.length > 0) request_body.excludeDomains = params.exclude_domains;
if (params.limit) request_body.maxResults = params.limit;
const raw_data = await http_json(this.name, `${config.ai_response.linkup.base_url}/search`, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(request_body),
signal: AbortSignal.timeout(config.ai_response.linkup.timeout)
});
const data = parse_provider_response(this.name, linkup_sourced_answer_response_schema, raw_data);
const results = [{
title: "Linkup AI Answer",
url: "",
snippet: data.answer,
score: 1,
source_provider: this.name,
metadata: {
type: "ai_answer",
depth: "standard",
sources_count: data.sources?.length || 0
}
}];
if (data.sources && data.sources.length > 0) {
const source_results = data.sources.map((source, index) => ({
title: source.name,
url: source.url,
snippet: source.snippet || "Source reference",
score: .9 - index * .01,
source_provider: this.name,
metadata: {
type: "source",
favicon: source.favicon
}
}));
results.push(...source_results);
}
return results;
} catch (error) {
handle_provider_error(error, this.name, "fetch AI response");
}
};
return retry_with_backoff(search_request);
}
};
//#endregion
//#region src/providers/ai-response/tavily-research/index.ts
const tavily_research_created_schema = v.object({
request_id: v.string(),
status: v.string(),
created_at: v.optional(v.string())
});
const tavily_research_status_schema = v.variant("status", [
v.object({
request_id: v.string(),
status: v.literal("completed"),
content: v.string(),
sources: v.array(v.object({
title: v.string(),
url: v.string(),
favicon: v.optional(v.string())
})),
response_time: v.number()
}),
v.object({
request_id: v.string(),
status: v.literal("failed"),
error: v.optional(v.string()),
response_time: v.optional(v.number())
}),
v.object({
request_id: v.string(),
status: v.picklist(["pending", "in_progress"]),
response_time: v.optional(v.number())
})
]);
const pending_result = (request_id, status) => [{
title: "Tavily Research Task",
url: "",
snippet: `Research is ${status}. Call ai_search again with provider "tavily_research" and research_id "${request_id}" to retrieve the report.`,
score: 1,
source_provider: "tavily_research",
metadata: {
type: "research_task",
request_id,
status
}
}];
var TavilyResearchProvider = class {
name = "tavily_research";
description = "Start or retrieve asynchronous Tavily research with synthesized reports and sources.";
async search(params) {
const api_key = validate_api_key(config.ai_response.tavily_research.api_key, this.name);
try {
if (params.research_id) {
const status = await this.get_status(params.research_id, api_key);
return this.map_status(status, params.limit);
}
const created = await retry_with_backoff(async () => {
const raw_created = await http_json(this.name, `${config.ai_response.tavily_research.base_url}/research`, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify({
input: sanitize_query(params.query),
model: "mini",
stream: false,
citation_format: "numbered",
output_length: "standard"
}),
signal: AbortSignal.timeout(config.ai_response.tavily_research.request_timeout)
});
return parse_provider_response(this.name, tavily_research_created_schema, raw_created);
}, {
max_retries: 1,
initial_delay: 250
});
return pending_result(created.request_id, created.status);
} catch (error) {
handle_provider_error(error, this.name, "complete research");
}
}
async get_status(research_id, api_key) {
return retry_with_backoff(async () => {
const raw_status = await http_json(this.name, `${config.ai_response.tavily_research.base_url}/research/${research_id}`, {
headers: { Authorization: `Bearer ${api_key}` },
expectedStatuses: [202],
signal: AbortSignal.timeout(config.ai_response.tavily_research.request_timeout)
});
return parse_provider_response(this.name, tavily_research_status_schema, raw_status);
}, {
max_retries: 1,
initial_delay: 250
});
}
map_status(status, limit) {
if (status.status === "failed") throw new ProviderError("PROVIDER_ERROR", status.error || "Tavily research task failed", this.name);
if (status.status !== "completed") return pending_result(status.request_id, status.status);
const results = [{
title: "Tavily Research Report",
url: "",
snippet: status.content,
score: 1,
source_provider: this.name,
metadata: {
type: "ai_answer",
request_id: status.request_id,
response_time: status.response_time,
sources_count: status.sources.length
}
}];
const source_limit = limit ?? status.sources.length;
results.push(...status.sources.slice(0, source_limit).map((source, index) => ({
title: source.title,
url: source.url,
snippet: "Source reference",
score: .9 - index * .01,
source_provider: this.name,
metadata: {
type: "source",
favicon: source.favicon
}
})));
return results;
}
};
//#endregion
//#region src/providers/enhancement/kagi-enrichment/index.ts
var KagiEnrichmentSearchProvider = class {
name = "kagi_enrichment";
description = "Search specialized indexes (Teclis for web, TinyGem for news). Ideal for discovering non-mainstream results and supplementary knowledge.";
async search(params) {
const api_key = validate_api_key(config.enhancement.kagi_enrichment.api_key, this.name);
const query = sanitize_query(params.query);
const limit = params.limit ?? 5;
const enrich_request = async () => {
try {
const [webData, newsData] = await Promise.all([http_json(this.name, `https://kagi.com/api/v0/enrich/web?${new URLSearchParams({
q: query,
limit: String(limit)
})}`, {
method: "GET",
headers: {
Authorization: `Bot ${api_key}`,
Accept: "application/json"
},
signal: AbortSignal.timeout(config.enhancement.kagi_enrichment.timeout)
}), http_json(this.name, `https://kagi.com/api/v0/enrich/news?${new URLSearchParams({
q: query,
limit: String(limit)
})}`, {
method: "GET",
headers: {
Authorization: `Bot ${api_key}`,
Accept: "application/json"
},
signal: AbortSignal.timeout(config.enhancement.kagi_enrichment.timeout)
})]);
if (!webData?.data || !newsData?.data) throw new ProviderError("API_ERROR", "Unexpected response: missing data from enrichment endpoints", this.name);
return [...webData.data, ...newsData.data].flatMap((result) => {
if (!result.title || !result.url) return [];
return [{
title: result.title,
url: result.url,
snippet: (result.snippet ?? "").replace(/'/g, "'").replace(/"/g, "\"").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">"),
score: result.rank ? 1 / result.rank : void 0,
source_provider: this.name
}];
});
} catch (error) {
handle_provider_error(error, this.name, "enrich content");
}
};
return retry_with_backoff(enrich_request);
}
};
//#endregion
//#region src/providers/processing/exa-contents/index.ts
const exa_contents_response_schema = v.object({
results: v.array(v.object({
id: v.string(),
title: v.string(),
url: v.string(),
text: v.optional(v.string()),
highlights: v.optional(v.array(v.string())),
summary: v.optional(v.string()),
publishedDate: v.optional(v.string()),
author: v.optional(v.string())
})),
requestId: v.string()
});
var ExaContentsProvider = class {
name = "exa_contents";
description = "Extract full content from Exa search result IDs";
async process_content(idsOrUrls, extract_depth = "basic") {
const api_key = validate_api_key(config.processing.exa_contents.api_key, this.name);
const items = Array.isArray(idsOrUrls) ? idsOrUrls : [idsOrUrls];
if (items.length === 0) throw new ProviderError("INVALID_INPUT", "At least one ID must be provided", this.name);
const process_request = async () => {
try {
const looksLikeUrl = (value) => {
try {
new URL(value);
return true;
} catch {
return false;
}
};
const request_body = {
...items.every(looksLikeUrl) ? { urls: items } : { ids: items },
text: true,
highlights: extract_depth === "advanced",
summary: extract_depth === "advanced"
};
const raw_data = await http_json(this.name, `${config.processing.exa_contents.base_url}/contents`, {
method: "POST",
headers: {
"x-api-key": api_key,
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(request_body)
});
const data = parse_provider_response(this.name, exa_contents_response_schema, raw_data);
let combined_content = "";
const raw_contents = [];
let total_word_count = 0;
for (const result of data.results) {
const content = result.text || result.summary || "No content available";
const word_count = content.split(/\s+/).length;
total_word_count += word_count;
combined_content += `## ${result.title}\n\n`;
if (result.author) combined_content += `**Author:** ${result.author}\n`;
if (result.publishedDate) combined_content += `**Published:** ${result.publishedDate}\n`;
combined_content += `**URL:** ${result.url}\n\n`;
if (result.highlights && result.highlights.length > 0) {
combined_content += `**Key Highlights:**\n`;
for (const highlight of result.highlights) combined_content += `- ${highlight}\n`;
combined_content += "\n";
}
if (result.summary && result.text) {
combined_content += `**Summary:** ${result.summary}\n\n`;
combined_content += `**Full Content:**\n${result.text}\n\n`;
} else combined_content += `${content}\n\n`;
combined_content += "---\n\n";
raw_contents.push({
url: result.url,
content
});
}
return {
content: combined_content,
raw_contents,
metadata: {
title: `Content from ${data.results.length} Exa results`,
word_count: total_word_count,
urls_processed: data.results.length,
successful_extractions: data.results.length,
extract_depth,
requestId: data.requestId
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "extract contents");
}
};
return retry_with_backoff(process_request);
}
};
//#endregion
//#region src/providers/processing/exa-similar/index.ts
const exa_similar_response_schema = v.object({
results: v.array(v.object({
id: v.string(),
title: v.string(),
url: v.string(),
text: v.optional(v.string()),
highlights: v.optional(v.array(v.string())),
summary: v.optional(v.string()),
publishedDate: v.optional(v.string()),
author: v.optional(v.string()),
score: v.optional(v.number())
})),
requestId: v.string()
});
var ExaSimilarProvider = class {
name = "exa_similar";
description = "Find web pages semantically similar to a given URL using Exa";
async process_content(url, extract_depth = "basic") {
const api_key = validate_api_key(config.processing.exa_similar.api_key, this.name);
const target_url = Array.isArray(url) ? url[0] : url;
if (!target_url) throw new ProviderError("INVALID_INPUT", "A URL must be provided", this.name);
validate_processing_urls(target_url, this.name);
const process_request = async () => {
try {
const request_body = {
url: target_url,
numResults: extract_depth === "advanced" ? 15 : 10,
contents: {
text: { maxCharacters: extract_depth === "advanced" ? 3e3 : 1500 },
highlights: extract_depth === "advanced",
summary: extract_depth === "advanced"
}
};
const raw_data = await http_json(this.name, `${config.processing.exa_similar.base_url}/findSimilar`, {
method: "POST",
headers: {
"x-api-key": api_key,
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(request_body)
});
const data = parse_provider_response(this.name, exa_similar_response_schema, raw_data);
let combined_content = `# Similar Pages to ${target_url}\n\n`;
combined_content += `Found ${data.results.length} similar pages:\n\n`;
const raw_contents = [];
let total_word_count = 0;
for (const result of data.results) {
const content = result.text || result.summary || "No content available";
const word_count = content.split(/\s+/).length;
total_word_count += word_count;
combined_content += `## ${result.title}\n\n`;
if (result.author) combined_content += `**Author:** ${result.author}\n`;
if (result.publishedDate) combined_content += `**Published:** ${result.publishedDate}\n`;
if (result.score) combined_content += `**Similarity Score:** ${result.score.toFixed(3)}\n`;
combined_content += `**URL:** ${result.url}\n\n`;
if (result.highlights && result.highlights.length > 0) {
combined_content += `**Key Highlights:**\n`;
for (const highlight of result.highlights) combined_content += `- ${highlight}\n`;
combined_content += "\n";
}
if (result.summary && result.text) {
combined_content += `**Summary:** ${result.summary}\n\n`;
combined_content += `**Content Preview:**\n${result.text.substring(0, 500)}${result.text.length > 500 ? "..." : ""}\n\n`;
} else combined_content += `${content.substring(0, 500)}${content.length > 500 ? "..." : ""}\n\n`;
combined_content += "---\n\n";
raw_contents.push({
url: result.url,
content
});
}
return {
content: combined_content,
raw_contents,
metadata: {
title: `Similar pages to ${target_url}`,
word_count: total_word_count,
urls_processed: data.results.length,
successful_extractions: data.results.length,
extract_depth,
original_url: target_url,
requestId: data.requestId
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "find similar pages");
}
};
return retry_with_backoff(process_request);
}
};
//#endregion
//#region src/common/firecrawl-utils.ts
const make_firecrawl_request = async (provider_name, base_url, api_key, body, timeout, schema) => {
const data = await http_json(provider_name, base_url, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(timeout)
});
return parse_provider_response(provider_name, schema, data);
};
const validate_firecrawl_response = (data, provider_name, error_message) => {
if (!data.success || data.error) throw new ProviderError("PROVIDER_ERROR", `${error_message}: ${data.error || "Unknown error"}`, provider_name);
};
const poll_firecrawl_job = async (config, schema) => {
let attempts = 0;
while (attempts < config.max_attempts) {
attempts++;
await new Promise((resolve) => setTimeout(resolve, config.poll_interval));
let status_result;
try {
const raw_status_result = await http_json(config.provider_name, config.status_url, {
method: "GET",
headers: { Authorization: `Bearer ${config.api_key}` },
signal: AbortSignal.timeout(config.timeout)
});
status_result = parse_provider_response(config.provider_name, schema, raw_status_result);
} catch (error) {
if (error instanceof ProviderError && error.details?.retryable === false) throw error;
continue;
}
if (status_result.success === false) throw new ProviderError("PROVIDER_ERROR", `Error checking job status: ${status_result.error || "Unknown error"}`, config.provider_name);
if (status_result.status === "completed" && status_result.data) return status_result;
if (status_result.status === "error" || status_result.status === "failed" || status_result.status === "cancelled") throw new ProviderError("PROVIDER_ERROR", `Job failed: ${status_result.error || "Unknown error"}`, config.provider_name);
}
throw new ProviderError("TIMEOUT", "Job timed out - try again later or with a smaller scope", config.provider_name, { retryable: true });
};
//#endregion
//#region src/providers/processing/firecrawl-actions/index.ts
const firecrawl_metadata_schema$2 = v.record(v.string(), v.unknown());
const firecrawl_actions_response_schema = v.object({
success: v.boolean(),
data: v.optional(v.object({
markdown: v.optional(v.string()),
html: v.optional(v.nullable(v.string())),
rawHtml: v.optional(v.nullable(v.string())),
screenshot: v.optional(v.nullable(v.string())),
actions: v.optional(v.object({ screenshots: v.optional(v.array(v.string())) })),
metadata: v.optional(firecrawl_metadata_schema$2)
})),
error: v.optional(v.string())
});
var FirecrawlActionsProvider = class {
name = "firecrawl_actions";
description = "Support for page interactions (clicking, scrolling, etc.) before extraction for dynamic content using Firecrawl. Enables extraction from JavaScript-heavy sites, single-page applications, and content behind user interactions. Best for accessing content that requires navigation, form filling, or other interactions.";
async process_content(url, extract_depth = "basic") {
const actions_url = validate_processing_urls(url, this.name)[0];
const actions_request = async () => {
const api_key = validate_api_key(config.processing.firecrawl_actions.api_key, this.name);
try {
const actions = extract_depth === "advanced" ? [
{
type: "wait",
milliseconds: 2e3
},
{
type: "scroll",
direction: "down"
},
{
type: "wait",
milliseconds: 1e3
},
{
type: "scroll",
direction: "down"
},
{
type: "wait",
milliseconds: 1e3
},
{
type: "click",
selector: "button:contains(\"Read more\"), button:contains(\"Show more\"), a:contains(\"Read more\"), a:contains(\"Show more\")"
},
{
type: "wait",
milliseconds: 2e3
}
] : [
{
type: "wait",
milliseconds: 2e3
},
{
type: "scroll",
direction: "down"
},
{
type: "wait",
milliseconds: 1e3
}
];
const actions_data = await make_firecrawl_request(this.name, config.processing.firecrawl_actions.base_url, api_key, {
url: actions_url,
formats: ["markdown", "screenshot"],
actions
}, config.processing.firecrawl_actions.timeout, firecrawl_actions_response_schema);
validate_firecrawl_response(actions_data, this.name, "Error performing actions");
if (!actions_data.data) throw new ProviderError("PROVIDER_ERROR", "No data returned from API", this.name);
if (!actions_data.data.markdown && !actions_data.data.html && !actions_data.data.rawHtml) throw new ProviderError("PROVIDER_ERROR", "No content extracted after performing actions", this.name);
const content = actions_data.data.markdown || actions_data.data.html || actions_data.data.rawHtml || "";
const actions_description = `# Content from ${actions_url} after interactions\n\nThe following actions were performed before extraction:\n\n` + actions.map((action, index) => {
switch (action.type) {
case "click": return `${index + 1}. Click on ${action.selector || `coordinates (${action.x}, ${action.y})`}`;
case "write": return `${index + 1}. Write "${action.text}" ${action.selector ? `into ${action.selector}` : ""}`;
case "scroll": return `${index + 1}. Scroll ${action.direction || "down"}`;
case "wait": return `${index + 1}. Wait ${action.milliseconds ? `for ${action.milliseconds}ms` : ""}`;
case "executeJavascript": return `${index + 1}. Execute JavaScript`;
case "screenshot": return `${index + 1}. Take screenshot`;
default: return `${index + 1}. Perform ${String(action.type)} action`;
}
}).join("\n") + "\n\n---\n\n" + content;
const raw_contents = [{
url: actions_url,
content: actions_description
}];
const word_count = actions_description.split(/\s+/).filter(Boolean).length;
return {
content: actions_description,
raw_contents,
metadata: {
title: `Content from ${actions_url} after interactions`,
word_count,
urls_processed: 1,
successful_extractions: 1,
extract_depth,
screenshot: actions_data.data.screenshot
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "perform actions");
}
};
return retry_with_backoff(actions_request);
}
};
//#endregion
//#region src/providers/processing/firecrawl-crawl/index.ts
const firecrawl_metadata_schema$1 = v.record(v.string(), v.unknown());
const firecrawl_crawl_response_schema = v.object({
success: v.boolean(),
id: v.string(),
url: v.string(),
error: v.optional(v.string())
});
const firecrawl_crawl_status_response_schema = v.object({
status: v.string(),
total: v.optional(v.number()),
completed: v.optional(v.number()),
data: v.optional(v.array(v.object({
url: v.optional(v.string()),
markdown: v.optional(v.string()),
html: v.optional(v.nullable(v.string())),
rawHtml: v.optional(v.nullable(v.string())),
metadata: v.optional(firecrawl_metadata_schema$1),
error: v.optional(v.nullable(v.string()))
}))),
error: v.optional(v.string())
});
const get_firecrawl_page_url = (page, fallback_url) => {
const metadata_url = page.metadata?.sourceURL ?? page.metadata?.url;
return page.url ?? (typeof metadata_url === "string" ? metadata_url : fallback_url);
};
const get_firecrawl_page_error = (page) => {
const metadata_error = page.metadata?.error;
return page.error ?? (typeof metadata_error === "string" ? metadata_error : void 0);
};
var FirecrawlCrawlProvider = class {
name = "firecrawl_crawl";
description = "Deep crawling of all accessible subpages on a website with configurable depth limits using Firecrawl. Efficiently discovers and extracts content from multiple pages within a domain. Best for comprehensive site analysis, content indexing, and data collection from entire websites.";
async process_content(url, extract_depth = "basic") {
const crawl_url = validate_processing_urls(url, this.name)[0];
const crawl_request = async () => {
const api_key = validate_api_key(config.processing.firecrawl_crawl.api_key, this.name);
try {
const crawl_data = await make_firecrawl_request(this.name, config.processing.firecrawl_crawl.base_url, api_key, {
url: crawl_url,
scrapeOptions: {
formats: ["markdown"],
onlyMainContent: true
},
maxDiscoveryDepth: extract_depth === "advanced" ? 3 : 1,
limit: extract_depth === "advanced" ? 50 : 20
}, config.processing.firecrawl_crawl.timeout, firecrawl_crawl_response_schema);
validate_firecrawl_response(crawl_data, this.name, "Error starting crawl");
const status_data = await poll_firecrawl_job({
provider_name: this.name,
status_url: `${config.processing.firecrawl_crawl.base_url}/${crawl_data.id}`,
api_key,
max_attempts: 20,
poll_interval: 5e3,
timeout: 3e4
}, firecrawl_crawl_status_response_schema);
if (!status_data.data || status_data.data.length === 0) throw new ProviderError("PROVIDER_ERROR", "Crawl returned no data", this.name);
const successful_pages = status_data.data.filter((page) => !get_firecrawl_page_error(page) && (page.markdown || page.html || page.rawHtml));
if (successful_pages.length === 0) throw new ProviderError("PROVIDER_ERROR", "All crawled pages failed to extract content", this.name);
const raw_contents = successful_pages.map((page) => ({
url: get_firecrawl_page_url(page, crawl_url),
content: page.markdown || page.html || page.rawHtml || ""
}));
const combined_content = raw_contents.map((result) => `# ${result.url}\n\n${result.content}\n\n---\n\n`).join("\n\n");
const word_count = combined_content.split(/\s+/).filter(Boolean).length;
const title_value = successful_pages[0]?.metadata?.title;
const title = typeof title_value === "string" ? title_value : void 0;
const failed_urls = status_data.data.filter((page) => get_firecrawl_page_error(page)).map((page) => get_firecrawl_page_url(page, crawl_url));
return {
content: combined_content,
raw_contents,
metadata: {
title,
word_count,
failed_urls: failed_urls.length > 0 ? failed_urls : void 0,
urls_processed: status_data.data.length,
successful_extractions: successful_pages.length,
extract_depth
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "crawl website");
}
};
return retry_with_backoff(crawl_request);
}
};
//#endregion
//#region src/providers/processing/firecrawl-extract/index.ts
const firecrawl_extract_response_schema = v.object({
success: v.boolean(),
id: v.string(),
error: v.optional(v.string())
});
const firecrawl_extract_status_response_schema = v.object({
success: v.boolean(),
id: v.string(),
status: v.string(),
data: v.optional(v.record(v.string(), v.unknown())),
error: v.optional(v.string())
});
var FirecrawlExtractProvider = class {
name = "firecrawl_extract";
description = "Structured data extraction with AI using natural language prompts via Firecrawl. Extracts specific information from web pages based on custom extraction instructions. Best for targeted data collection, information extraction, and converting unstructured web content into structured data.";
async process_content(url, extract_depth = "basic") {
const extract_url = validate_processing_urls(url, this.name)[0];
const extract_request = async () => {
const api_key = validate_api_key(config.processing.firecrawl_extract.api_key, this.name);
try {
const extraction_prompt = extract_depth === "advanced" ? "Extract all relevant information from this page including: title, author, date published, main content, categories or tags, related links, and any structured data like product information, pricing, or specifications. Format the data in a well-structured way." : "Extract the main content, title, and author from this page. Summarize the key information.";
const extract_data = await make_firecrawl_request(this.name, config.processing.firecrawl_extract.base_url, api_key, {
urls: [extract_url],
prompt: extraction_prompt,
showSources: true,
scrapeOptions: {
formats: ["markdown"],
onlyMainContent: true,
waitFor: extract_depth === "advanced" ? 5e3 : 2e3
}
}, config.processing.firecrawl_extract.timeout, firecrawl_extract_response_schema);
validate_firecrawl_response(extract_data, this.name, "Error starting extraction");
const status_data = await poll_firecrawl_job({
provider_name: this.name,
status_url: `${config.processing.firecrawl_extract.base_url}/${extract_data.id}`,
api_key,
max_attempts: 15,
poll_interval: 3e3,
timeout: 3e4
}, firecrawl_extract_status_response_schema);
if (!status_data.data) throw new ProviderError("PROVIDER_ERROR", "No data extracted from URL", this.name);
let formatted_content = `# Extracted Data from ${extract_url}\n\n`;
for (const [key, value] of Object.entries(status_data.data)) if (typeof value === "string") formatted_content += `## ${key.charAt(0).toUpperCase() + key.slice(1)}\n\n${value}\n\n`;
else if (Array.isArray(value)) {
formatted_content += `## ${key.charAt(0).toUpperCase() + key.slice(1)}\n\n`;
value.forEach((item, index) => {
if (typeof item === "object") {
formatted_content += `### Item ${index + 1}\n\n`;
for (const [itemKey, itemValue] of Object.entries(item)) formatted_content += `- **${itemKey}**: ${String(itemValue)}\n`;
formatted_content += "\n";
} else formatted_content += `- ${item}\n`;
});
formatted_content += "\n";
} else if (typeof value === "object" && value !== null) {
formatted_content += `## ${key.charAt(0).toUpperCase() + key.slice(1)}\n\n`;
for (const [subKey, subValue] of Object.entries(value)) formatted_content += `- **${subKey}**: ${subValue}\n`;
formatted_content += "\n";
}
const raw_contents = [{
url: extract_url,
content: formatted_content
}];
const title = typeof status_data.data.title === "string" ? status_data.data.title : `Extracted Data from ${extract_url}`;
const word_count = formatted_content.split(/\s+/).filter(Boolean).length;
return {
content: formatted_content,
raw_contents,
metadata: {
title,
word_count,
urls_processed: 1,
successful_extractions: 1,
extract_depth
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "extract data");
}
};
return retry_with_backoff(extract_request);
}
};
//#endregion
//#region src/providers/processing/firecrawl-map/index.ts
const firecrawl_map_response_schema = v.object({
success: v.boolean(),
links: v.optional(v.array(v.object({
url: v.string(),
title: v.optional(v.string())
}))),
error: v.optional(v.string())
});
var FirecrawlMapProvider = class {
name = "firecrawl_map";
description = "Fast URL collection from websites for comprehensive site mapping using Firecrawl. Efficiently discovers all accessible URLs within a domain without extracting content. Best for site auditing, URL discovery, and preparing for targeted content extraction.";
async process_content(url, extract_depth = "basic") {
const map_url = validate_processing_urls(url, this.name)[0];
const map_request = async () => {
const api_key = validate_api_key(config.processing.firecrawl_map.api_key, this.name);
try {
const map_data = await make_firecrawl_request(this.name, config.processing.firecrawl_map.base_url, api_key, {
url: map_url,
limit: extract_depth === "advanced" ? 200 : 50,
includeSubdomains: false
}, config.processing.firecrawl_map.timeout, firecrawl_map_response_schema);
validate_firecrawl_response(map_data, this.name, "Error mapping website");
if (!map_data.links || map_data.links.length === 0) throw new ProviderError("PROVIDER_ERROR", "No URLs discovered during mapping", this.name);
const formatted_content = `# Site Map for ${map_url}\n\nFound ${map_data.links.length} URLs:\n\n` + map_data.links.map((link) => link.title ? `- ${link.url} — ${link.title}` : `- ${link.url}`).join("\n");
return {
content: formatted_content,
raw_contents: [{
url: map_url,
content: formatted_content
}],
metadata: {
title: `Site Map for ${map_url}`,
word_count: map_data.links.length,
urls_processed: 1,
successful_extractions: 1,
extract_depth
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "map website");
}
};
return retry_with_backoff(map_request);
}
};
//#endregion
//#region src/common/results.ts
const CHARS_PER_TOKEN = 4;
const MAX_SAFE_CHARS = 2e4 * CHARS_PER_TOKEN;
const get_large_result_mode = (mode_override) => {
if (mode_override) return mode_override;
const configured_mode = process.env.OMNISEARCH_LARGE_RESULT_MODE;
if (configured_mode === "inline" || configured_mode === "file") return configured_mode;
return "file";
};
const format_as_text = (result) => {
const lines = [];
const sections = [];
let current_line = 1;
const add_line = (line) => {
if (line.startsWith("URL: ")) sections.push({
title: line,
line: current_line
});
else if (line.startsWith("# ")) sections.push({
title: line.slice(2),
line: current_line
});
else if (line.startsWith("## ")) sections.push({
title: line.slice(3),
line: current_line
});
else if (line.startsWith("### ")) sections.push({
title: line.slice(4),
line: current_line
});
lines.push(line);
current_line++;
};
const add_content = (content) => {
const content_lines = content.split("\n");
for (const line of content_lines) add_line(line);
};
const raw_contents = result.raw_contents;
if (raw_contents?.length) for (const item of raw_contents) {
add_line("=".repeat(80));
add_line(`URL: ${item.url}`);
add_line("=".repeat(80));
if (item.content) add_content(item.content);
add_line("");
}
else if (result.content) add_content(JSON.stringify(result.content));
else add_content(JSON.stringify(result, null, 2));
if (result.metadata) {
add_line("");
add_line("=".repeat(80));
sections.push({
title: "METADATA",
line: current_line
});
add_line("METADATA");
add_line("=".repeat(80));
add_content(JSON.stringify(result.metadata, null, 2));
}
return {
text: lines.join("\n"),
sections,
total_lines: current_line - 1
};
};
const handle_large_result = (result, provider_name, options = {}) => {
const char_count = JSON.stringify(result, null, 2).length;
if (char_count <= MAX_SAFE_CHARS) return result;
if (get_large_result_mode(options.mode) === "inline") return result;
const file_id = randomUUID();
const file_path = join(tmpdir(), `mcp-${provider_name}-${file_id}.txt`);
const { text, sections, total_lines } = format_as_text(result);
writeFileSync(file_path, text, "utf-8");
const result_obj = result;
const metadata = result_obj.metadata;
const word_count = metadata?.word_count ?? "unknown";
const urls_processed = metadata?.urls_processed ?? "unknown";
return {
file_path,
total_lines,
estimated_tokens: Math.round(char_count / CHARS_PER_TOKEN),
sections,
read_hint: `Use Read tool with file_path="${file_path}" and offset=LINE_NUMBER limit=50 to read a section`,
metadata: {
word_count,
urls_processed,
source_provider: result_obj.source_provider
}
};
};
const omit_raw_contents = (result) => {
const { raw_contents: _raw_contents, ...compact_result } = result;
return compact_result;
};
const aggregate_url_results = (results, provider_name, urls, extract_depth) => {
const successful_results = results.filter((r) => r.success);
const failed_urls = results.filter((r) => !r.success).map((r) => r.url);
if (successful_results.length === 0) throw new ProviderError("PROVIDER_ERROR", "Failed to extract content from all URLs", provider_name);
const raw_contents = successful_results.map((result) => ({
url: result.url,
content: result.content
}));
const combined_content = raw_contents.map((result) => result.content).join("\n\n");
const word_count = combined_content.split(/\s+/).filter(Boolean).length;
return {
content: combined_content,
raw_contents,
metadata: {
title: successful_results[0]?.metadata?.title,
word_count,
failed_urls: failed_urls.length > 0 ? failed_urls : void 0,
urls_processed: urls.length,
successful_extractions: successful_results.length,
extract_depth
},
source_provider: provider_name
};
};
//#endregion
//#region src/providers/processing/firecrawl-scrape/index.ts
const firecrawl_metadata_schema = v.record(v.string(), v.unknown());
const firecrawl_scrape_response_schema = v.object({
success: v.boolean(),
data: v.optional(v.object({
markdown: v.optional(v.string()),
html: v.optional(v.nullable(v.string())),
rawHtml: v.optional(v.nullable(v.string())),
screenshot: v.optional(v.nullable(v.string())),
links: v.optional(v.array(v.string())),
metadata: v.optional(firecrawl_metadata_schema),
llm_extraction: v.optional(v.unknown()),
warning: v.optional(v.string())
})),
error: v.optional(v.string())
});
var FirecrawlScrapeProvider = class {
name = "firecrawl_scrape";
description = "Extract clean, LLM-ready data from single URLs with enhanced formatting options using Firecrawl. Efficiently converts web content into markdown, plain text, or structured data with configurable extraction options. Best for content analysis, data collection, and AI training data preparation.";
async process_content(url, extract_depth = "basic") {
const urls = validate_processing_urls(url, this.name);
const scrape_request = async () => {
const api_key = validate_api_key(config.processing.firecrawl_scrape.api_key, this.name);
try {
const results = await Promise.all(urls.map(async (single_url) => {
try {
const data = await make_firecrawl_request(this.name, config.processing.firecrawl_scrape.base_url, api_key, {
url: single_url,
formats: ["markdown"],
onlyMainContent: true,
waitFor: extract_depth === "advanced" ? 5e3 : 2e3
}, config.processing.firecrawl_scrape.timeout, firecrawl_scrape_response_schema);
validate_firecrawl_response(data, this.name, "Error scraping URL");
if (!data.data) throw new ProviderError("PROVIDER_ERROR", "No data returned from API", this.name);
if (!data.data.markdown && !data.data.html && !data.data.rawHtml) throw new ProviderError("PROVIDER_ERROR", "No content extracted from URL", this.name);
return {
url: single_url,
content: data.data.markdown || data.data.html || data.data.rawHtml || "",
metadata: data.data.metadata,
success: true
};
} catch (error) {
console.error(`Error processing ${single_url}:`, error);
return {
url: single_url,
content: "",
success: false,
error: error instanceof Error ? error.message : "Unknown error"
};
}
}));
return aggregate_url_results(results, this.name, urls, extract_depth);
} catch (error) {
handle_provider_error(error, this.name, "extract content");
}
};
return retry_with_backoff(scrape_request);
}
};
//#endregion
//#region src/providers/processing/kagi-summarizer/index.ts
const kagi_summarizer_response_schema = v.object({
meta: v.object({
id: v.string(),
node: v.string(),
ms: v.number(),
api_balance: v.optional(v.number())
}),
data: v.object({
output: v.string(),
tokens: v.number()
})
});
var KagiSummarizerProvider = class {
name = "kagi_summarizer";
description = "Instantly summarizes content of any type and length from URLs. Supports pages, videos, and podcasts with transcripts. Best for quick comprehension of long-form content and multimedia resources.";
async process_content(url) {
const api_key = validate_api_key(config.processing.kagi_summarizer.api_key, this.name);
const summarize_request = async () => {
try {
const raw_data = await http_json(this.name, config.processing.kagi_summarizer.base_url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bot ${api_key}`
},
body: JSON.stringify({ url }),
signal: AbortSignal.timeout(config.processing.kagi_summarizer.timeout)
});
const data = parse_provider_response(this.name, kagi_summarizer_response_schema, raw_data);
return {
content: data.data.output,
metadata: { word_count: data.data.tokens },
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "fetch summary");
}
};
return retry_with_backoff(summarize_request);
}
};
//#endregion
//#region src/providers/processing/tavily-crawl/index.ts
const tavily_crawl_response_schema = v.object({
base_url: v.string(),
results: v.array(v.object({
url: v.string(),
raw_content: v.nullable(v.string()),
favicon: v.optional(v.nullable(v.string()))
})),
response_time: v.union([v.number(), v.string()]),
request_id: v.optional(v.string()),
usage: v.optional(v.object({ credits: v.number() }))
});
var TavilyCrawlProvider = class {
name = "tavily_crawl";
description = "Crawl a website with Tavily graph-based discovery and content extraction. Best for documentation indexing and multi-page site analysis.";
async process_content(url, extract_depth = "basic") {
const [crawl_url] = validate_processing_urls(url, this.name);
const crawl_request = async () => {
const api_key = validate_api_key(config.processing.tavily_crawl.api_key, this.name);
try {
const raw_data = await http_json(this.name, `${config.processing.tavily_crawl.base_url}/crawl`, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify({
url: crawl_url,
max_depth: extract_depth === "advanced" ? 3 : 1,
max_breadth: extract_depth === "advanced" ? 50 : 20,
limit: extract_depth === "advanced" ? 50 : 20,
extract_depth,
format: "markdown",
include_favicon: true,
include_usage: true
}),
signal: AbortSignal.timeout(config.processing.tavily_crawl.timeout)
});
const data = parse_provider_response(this.name, tavily_crawl_response_schema, raw_data);
const successful_results = data.results.filter((result) => typeof result.raw_content === "string" && result.raw_content.length > 0);
if (successful_results.length === 0) throw new ProviderError("PROVIDER_ERROR", "Crawl returned no content", this.name);
const raw_contents = successful_results.map((result) => ({
url: result.url,
content: result.raw_content
}));
const content = raw_contents.map((result) => `# ${result.url}\n\n${result.content}`).join("\n\n---\n\n");
const favicons = Object.fromEntries(successful_results.filter((result) => result.favicon).map((result) => [result.url, result.favicon]));
const failed_urls = data.results.filter((result) => !result.raw_content).map((result) => result.url);
return {
content,
raw_contents,
metadata: {
word_count: content.split(/\s+/).filter(Boolean).length,
urls_processed: data.results.length,
successful_extractions: successful_results.length,
failed_urls: failed_urls.length > 0 ? failed_urls : void 0,
extract_depth,
response_time: data.response_time,
...data.request_id ? { request_id: data.request_id } : {},
...data.usage ? { usage: data.usage } : {},
...Object.keys(favicons).length > 0 ? { favicons } : {}
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "crawl website");
}
};
return retry_with_backoff(crawl_request);
}
};
//#endregion
//#region src/providers/processing/tavily-extract/index.ts
const tavily_extract_response_schema = v.object({
results: v.array(v.object({
url: v.string(),
raw_content: v.string(),
images: v.optional(v.array(v.string())),
favicon: v.optional(v.string())
})),
failed_results: v.array(v.object({
url: v.string(),
error: v.string()
})),
response_time: v.union([v.number(), v.string()]),
request_id: v.optional(v.string()),
usage: v.optional(v.object({ credits: v.number() }))
});
var TavilyExtractProvider = class {
name = "tavily_extract";
description = "Extract web page content from single or multiple URLs using Tavily Extract. Efficiently converts web content into clean, processable text with configurable extraction depth and optional image extraction. Returns both combined and individual URL content. Best for content analysis, data collection, and research.";
async process_content(url, extract_depth = "basic", options = {}) {
const urls = validate_processing_urls(url, this.name);
if (options.chunks_per_source !== void 0 && !options.query) throw new ProviderError("INVALID_INPUT", "query is required when chunks_per_source is provided", this.name);
const extract_request = async () => {
const api_key = validate_api_key(config.processing.tavily_extract.api_key, this.name);
try {
const request_body = {
urls,
include_images: false,
include_favicon: true,
include_usage: true,
extract_depth,
format: options.format ?? "markdown",
...options.query ? { query: sanitize_query(options.query) } : {},
...options.chunks_per_source !== void 0 ? { chunks_per_source: options.chunks_per_source } : {}
};
const raw_data = await http_json(this.name, `${config.processing.tavily_extract.base_url}/extract`, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(request_body),
signal: AbortSignal.timeout(config.processing.tavily_extract.timeout)
});
const data = parse_provider_response(this.name, tavily_extract_response_schema, raw_data);
if (data.results.length === 0) throw new ProviderError("PROVIDER_ERROR", "No content extracted from URL", this.name);
const raw_contents = data.results.map((result) => ({
url: result.url,
content: result.raw_content
}));
const combined_content = raw_contents.map((result) => result.content).join("\n\n");
const word_count = combined_content.split(/\s+/).filter(Boolean).length;
const favicons = Object.fromEntries(data.results.filter((result) => result.favicon).map((result) => [result.url, result.favicon]));
return {
content: combined_content,
raw_contents,
metadata: {
word_count,
failed_urls: data.failed_results.length > 0 ? data.failed_results.map((f) => f.url) : void 0,
urls_processed: urls.length,
successful_extractions: data.results.length,
extract_depth,
response_time: data.response_time,
...data.request_id ? { request_id: data.request_id } : {},
...data.usage ? { usage: data.usage } : {},
...Object.keys(favicons).length > 0 ? { favicons } : {}
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "extract content");
}
};
return retry_with_backoff(extract_request);
}
};
//#endregion
//#region src/providers/processing/tavily-map/index.ts
const tavily_map_response_schema = v.object({
base_url: v.string(),
results: v.array(v.string()),
response_time: v.union([v.number(), v.string()]),
request_id: v.optional(v.string()),
usage: v.optional(v.object({ credits: v.number() }))
});
var TavilyMapProvider = class {
name = "tavily_map";
description = "Discover URLs and site structure with Tavily Map. Best for fast site discovery before targeted extraction or crawling.";
async process_content(url, extract_depth = "basic") {
const [map_url] = validate_processing_urls(url, this.name);
const map_request = async () => {
const api_key = validate_api_key(config.processing.tavily_map.api_key, this.name);
try {
const raw_data = await http_json(this.name, `${config.processing.tavily_map.base_url}/map`, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify({
url: map_url,
max_depth: extract_depth === "advanced" ? 3 : 1,
max_breadth: extract_depth === "advanced" ? 50 : 20,
limit: extract_depth === "advanced" ? 200 : 50,
include_usage: true
}),
signal: AbortSignal.timeout(config.processing.tavily_map.timeout)
});
const data = parse_provider_response(this.name, tavily_map_response_schema, raw_data);
if (data.results.length === 0) throw new ProviderError("PROVIDER_ERROR", "No URLs discovered during mapping", this.name);
const content = `# Site Map for ${map_url}\n\n` + data.results.map((result) => `- ${result}`).join("\n");
return {
content,
raw_contents: [{
url: map_url,
content
}],
metadata: {
title: `Site Map for ${map_url}`,
word_count: data.results.length,
urls_processed: 1,
successful_extractions: data.results.length,
extract_depth,
response_time: data.response_time,
...data.request_id ? { request_id: data.request_id } : {},
...data.usage ? { usage: data.usage } : {}
},
source_provider: this.name
};
} catch (error) {
handle_provider_error(error, this.name, "map website");
}
};
return retry_with_backoff(map_request);
}
};
//#endregion
//#region src/common/search-operators.ts
const operator_patterns = {
exclude_site: /-site:([^\s]+)/g,
site: /site:([^\s]+)/g,
filetype: /filetype:([^\s]+)/g,
ext: /ext:([^\s]+)/g,
intitle: /intitle:([^\s]+)/g,
inurl: /inurl:([^\s]+)/g,
inbody: /inbody:"?([^"\s]+)"?/g,
inpage: /inpage:"?([^"\s]+)"?/g,
language: /(?:lang|language):([^\s]+)/g,
location: /(?:loc|location):([^\s]+)/g,
before: /before:(\d{4}(?:-\d{2}(?:-\d{2})?)?)/g,
after: /after:(\d{4}(?:-\d{2}(?:-\d{2})?)?)/g,
exact: /"([^"]+)"/g,
force_include: /\+([^\s]+)/g,
exclude_term: /-([^\s:]+)(?!\s*site:)/g,
boolean: /\b(AND|OR|NOT)\b/g
};
const parse_search_operators = (query) => {
const operators = [];
let modified_query = query;
Object.entries(operator_patterns).forEach(([type, pattern]) => {
modified_query = modified_query.replace(pattern, (match, value) => {
operators.push({
type,
value,
original_text: match
});
return "";
});
});
return {
base_query: modified_query.replace(/\s+/g, " ").trim(),
operators
};
};
const apply_search_operators = (parsed_query) => {
const params = { query: parsed_query.base_query };
for (const operator of parsed_query.operators) switch (operator.type) {
case "site":
params.include_domains = [...params.include_domains || [], operator.value];
break;
case "exclude_site":
params.exclude_domains = [...params.exclude_domains || [], operator.value];
break;
case "filetype":
case "ext":
params.file_type = operator.value;
break;
case "intitle":
params.title_filter = operator.value;
break;
case "inurl":
params.url_filter = operator.value;
break;
case "inbody":
params.body_filter = operator.value;
break;
case "inpage":
params.page_filter = operator.value;
break;
case "language":
params.language = operator.value;
break;
case "location":
params.location = operator.value;
break;
case "before":
params.date_before = operator.value;
break;
case "after":
params.date_after = operator.value;
break;
case "exact":
params.exact_phrases = [...params.exact_phrases || [], operator.value];
break;
case "force_include":
params.force_include_terms = [...params.force_include_terms || [], operator.value];
break;
case "exclude_term":
params.exclude_terms = [...params.exclude_terms || [], operator.value];
break;
case "boolean":
if (!params.boolean_operators) params.boolean_operators = [];
params.boolean_operators.push({
type: operator.value,
terms: []
});
}
return params;
};
const build_query_with_operators = (search_params, additional_include_domains, additional_exclude_domains, options) => {
let query = search_params.query;
const filters = [];
const include_domains = [...additional_include_domains ?? [], ...search_params.include_domains ?? []];
if (include_domains.length) {
const domain_filter = include_domains.map((domain) => `site:${domain}`).join(" OR ");
filters.push(domain_filter);
}
const exclude_domains = [...additional_exclude_domains ?? [], ...search_params.exclude_domains ?? []];
if (exclude_domains.length) filters.push(...exclude_domains.map((domain) => `-site:${domain}`));
if (search_params.file_type && !options?.exclude_file_type) filters.push(`filetype:${search_params.file_type}`);
if (search_params.title_filter) filters.push(`intitle:${search_params.title_filter}`);
if (search_params.url_filter) filters.push(`inurl:${search_params.url_filter}`);
if (search_params.body_filter) filters.push(`inbody:${search_params.body_filter}`);
if (search_params.page_filter) filters.push(`inpage:${search_params.page_filter}`);
if (search_params.language) filters.push(`lang:${search_params.language}`);
if (search_params.location) filters.push(`loc:${search_params.location}`);
if (search_params.date_before && !options?.exclude_dates) filters.push(`before:${search_params.date_before}`);
if (search_params.date_after && !options?.exclude_dates) filters.push(`after:${search_params.date_after}`);
if (search_params.exact_phrases?.length) filters.push(...search_params.exact_phrases.map((phrase) => `"${phrase}"`));
if (search_params.force_include_terms?.length) filters.push(...search_params.force_include_terms.map((term) => `+${term}`));
if (search_params.exclude_terms?.length) filters.push(...search_params.exclude_terms.map((term) => `-${term}`));
if (filters.length > 0) query = `${query} ${filters.join(" ")}`;
return query;
};
//#endregion
//#region src/providers/search/brave/index.ts
const brave_search_response_schema = v.object({ web: v.optional(v.object({ results: v.array(v.object({
title: v.string(),
url: v.string(),
description: v.optional(v.string())
})) })) });
var BraveSearchProvider = class {
name = "brave";
description = "Privacy-focused search with operators: site:, -site:, filetype:/ext:, intitle:, inurl:, inbody:, inpage:, lang:, loc:, before:, after:, +term, -term, \"exact\". Best for technical content and privacy-sensitive queries.";
async search(params) {
const api_key = validate_api_key(config.search.brave.api_key, this.name);
const parsed_query = parse_search_operators(params.query);
const search_params = apply_search_operators(parsed_query);
const search_request = async () => {
try {
const query = build_query_with_operators(search_params, params.include_domains, params.exclude_domains);
const query_params = new URLSearchParams({
q: query,
count: (params.limit ?? 10).toString()
});
const raw_data = await http_json(this.name, `${config.search.brave.base_url}/web/search?${query_params}`, {
method: "GET",
headers: {
Accept: "application/json",
"X-Subscription-Token": api_key
},
signal: AbortSignal.timeout(config.search.brave.timeout)
});
return (parse_provider_response(this.name, brave_search_response_schema, raw_data).web?.results ?? []).map((result) => ({
title: result.title,
url: result.url,
snippet: result.description ?? "",
source_provider: this.name
}));
} catch (error) {
handle_provider_error(error, this.name, "fetch search results");
}
};
return retry_with_backoff(search_request);
}
};
//#endregion
//#region src/providers/search/exa/index.ts
const exa_search_response_schema = v.object({
requestId: v.string(),
autopromptString: v.optional(v.string()),
resolvedSearchType: v.optional(v.string()),
searchType: v.optional(v.string()),
results: v.array(v.object({
id: v.string(),
title: v.string(),
url: v.string(),
publishedDate: v.optional(v.string()),
author: v.optional(v.string()),
text: v.optional(v.string()),
score: v.optional(v.number()),
highlights: v.optional(v.array(v.string())),
summary: v.optional(v.string())
}))
});
var ExaSearchProvider = class {
name = "exa";
description = "AI-powered web search using neural and keyword search. Optimized for AI applications with semantic understanding, content extraction, and research capabilities.";
async search(params) {
const api_key = validate_api_key(config.search.exa.api_key, this.name);
const search_request = async () => {
try {
const request_body = {
query: sanitize_query(params.query),
type: "auto",
numResults: params.limit ?? 10,
contents: { text: { maxCharacters: 3e3 } }
};
if (params.include_domains && params.include_domains.length > 0) request_body.includeDomains = params.include_domains;
if (params.exclude_domains && params.exclude_domains.length > 0) request_body.excludeDomains = params.exclude_domains;
const raw_data = await http_json(this.name, `${config.search.exa.base_url}/search`, {
method: "POST",
headers: {
"x-api-key": api_key,
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(request_body)
});
const data = parse_provider_response(this.name, exa_search_response_schema, raw_data);
return data.results.map((result) => ({
title: result.title,
url: result.url,
snippet: result.text || result.summary || "No content available",
score: result.score || 0,
source_provider: this.name,
metadata: {
id: result.id,
author: result.author,
publishedDate: result.publishedDate,
highlights: result.highlights,
autopromptString: data.autopromptString,
resolvedSearchType: data.resolvedSearchType ?? data.searchType
}
}));
} catch (error) {
handle_provider_error(error, this.name, "fetch search results");
}
};
return retry_with_backoff(search_request);
}
};
//#endregion
//#region src/providers/search/github/index.ts
const github_code_search_response_schema = v.object({ items: v.array(v.object({
name: v.string(),
path: v.string(),
html_url: v.string(),
score: v.number(),
repository: v.object({
full_name: v.string(),
html_url: v.string()
}),
text_matches: v.optional(v.array(v.object({ fragment: v.optional(v.string()) })))
})) });
const github_repository_search_response_schema = v.object({ items: v.array(v.object({
full_name: v.string(),
html_url: v.string(),
description: v.nullable(v.string()),
stargazers_count: v.number(),
forks_count: v.number(),
pushed_at: v.string(),
language: v.nullable(v.string()),
score: v.number()
})) });
const github_user_search_response_schema = v.object({ items: v.array(v.object({
login: v.string(),
html_url: v.string(),
bio: v.optional(v.nullable(v.string())),
type: v.string(),
score: v.number()
})) });
const is_github_search_error = (error) => typeof error === "object" && error !== null;
var GitHubSearchProvider = class {
name = "github";
description = "Search for code on GitHub. This is ideal for finding code examples, tracking down function definitions, or locating files with specific names or paths. Supports advanced query syntax with qualifiers like `filename:`, `path:`, `repo:`, `user:`, `language:`, and `in:file`. For example, to find a file named `settings.json` in a `.claude` directory, you could use the query: `filename:settings.json path:.claude`";
async search(params) {
return this.search_code(params);
}
async search_code(params) {
const api_key = validate_api_key(config.search.github.api_key, this.name);
const octokit = new Octokit({ auth: api_key });
const search_request = async () => {
try {
const response = await octokit.rest.search.code({
q: params.query,
per_page: params.limit ?? 10,
headers: { accept: "application/vnd.github.v3.text-match+json" }
});
return parse_provider_response(this.name, github_code_search_response_schema, response.data).items.map((item) => {
let snippet = `No snippet available for ${item.path}`;
if (item.text_matches && item.text_matches.length > 0) {
const fragments = item.text_matches.map((match) => match.fragment).filter(Boolean);
if (fragments.length > 0) snippet = fragments.slice(0, 2).join(" ... ");
}
return {
title: `${item.repository.full_name}/${item.path}`,
url: item.html_url,
snippet,
score: item.score,
source_provider: this.name,
metadata: {
repository: item.repository.full_name,
file_path: item.path,
file_name: item.name,
search_type: "code"
}
};
});
} catch (error) {
return this.handle_search_error(error);
}
};
return retry_with_backoff(search_request);
}
async search_repositories(params) {
const api_key = validate_api_key(config.search.github.api_key, this.name);
const octokit = new Octokit({ auth: api_key });
const search_request = async () => {
try {
const response = await octokit.rest.search.repos({
q: params.query,
per_page: params.limit ?? 10,
sort: params.sort
});
return parse_provider_response(this.name, github_repository_search_response_schema, response.data).items.map((item) => {
let snippet = item.description ?? "No description available.";
if (item.language) snippet += ` • Language: ${item.language}`;
snippet += ` • ⭐ ${item.stargazers_count} • 🍴 ${item.forks_count}`;
return {
title: item.full_name,
url: item.html_url,
snippet,
score: item.score,
source_provider: this.name,
metadata: {
repository: item.full_name,
language: item.language,
stars: item.stargazers_count,
forks: item.forks_count,
last_push: item.pushed_at,
search_type: "repository"
}
};
});
} catch (error) {
return this.handle_search_error(error);
}
};
return retry_with_backoff(search_request);
}
async repository_search(params) {
return this.search_repositories(params);
}
async search_users(params) {
const api_key = validate_api_key(config.search.github.api_key, this.name);
const octokit = new Octokit({ auth: api_key });
const search_request = async () => {
try {
const response = await octokit.rest.search.users({
q: params.query,
per_page: params.limit ?? 10
});
return parse_provider_response(this.name, github_user_search_response_schema, response.data).items.map((user) => ({
title: user.login,
url: user.html_url,
snippet: user.bio ?? `GitHub user: ${user.login} • ${user.type}`,
score: user.score,
source_provider: this.name,
metadata: {
username: user.login,
user_type: user.type,
search_type: "user"
}
}));
} catch (error) {
return this.handle_search_error(error);
}
};
return retry_with_backoff(search_request);
}
handle_search_error(error) {
const status = is_github_search_error(error) && typeof error.status === "number" ? error.status : 500;
const message = is_github_search_error(error) && typeof error.message === "string" ? error.message : "An unexpected error occurred.";
throw normalize_provider_http_error(this.name, status, message);
}
};
new GitHubSearchProvider();
//#endregion
//#region src/providers/search/kagi/index.ts
const kagi_search_response_schema = v.object({
data: v.array(v.unknown()),
meta: v.optional(v.object({
total_hits: v.optional(v.number()),
api_balance: v.optional(v.number())
}))
});
const is_kagi_search_result = (result) => typeof result === "object" && result !== null && "title" in result && "url" in result && typeof result.title === "string" && typeof result.url === "string" && (!("snippet" in result) || typeof result.snippet === "string" || result.snippet === null) && (!("rank" in result) || typeof result.rank === "number");
var KagiSearchProvider = class {
name = "kagi";
description = "High-quality search with operators: site:, -site:, filetype:/ext:, intitle:, inurl:, inbody:, inpage:, lang:, loc:, before:, after:, +term, -term, \"exact\". Privacy-focused with specialized knowledge indexes. Best for research and technical documentation.";
async search(params) {
const api_key = validate_api_key(config.search.kagi.api_key, this.name);
const parsed_query = parse_search_operators(params.query);
const search_params = apply_search_operators(parsed_query);
const search_request = async () => {
try {
const query = build_query_with_operators(search_params, params.include_domains, params.exclude_domains, {
exclude_file_type: true,
exclude_dates: true
});
const query_params = new URLSearchParams({
q: query,
limit: (params.limit ?? 10).toString()
});
if (search_params.file_type) query_params.append("file_type", search_params.file_type);
if (search_params.date_before || search_params.date_after) {
const time_range = [];
if (search_params.date_after) time_range.push(`after:${search_params.date_after}`);
if (search_params.date_before) time_range.push(`before:${search_params.date_before}`);
query_params.append("time_range", time_range.join(","));
}
const raw_data = await http_json(this.name, `${config.search.kagi.base_url}/search?${query_params}`, {
method: "GET",
headers: {
Authorization: `Bot ${api_key}`,
Accept: "application/json"
},
signal: AbortSignal.timeout(config.search.kagi.timeout)
});
return parse_provider_response(this.name, kagi_search_response_schema, raw_data).data.filter(is_kagi_search_result).map((result) => ({
title: result.title,
url: result.url,
snippet: result.snippet ?? "",
score: result.rank,
source_provider: this.name
}));
} catch (error) {
handle_provider_error(error, this.name, "fetch search results");
}
};
return retry_with_backoff(search_request);
}
};
//#endregion
//#region src/providers/search/tavily/index.ts
const tavily_search_response_schema = v.object({
results: v.array(v.object({
title: v.string(),
url: v.string(),
content: v.string(),
raw_content: v.optional(v.nullable(v.string())),
favicon: v.optional(v.string()),
score: v.number()
})),
response_time: v.optional(v.union([v.number(), v.string()])),
request_id: v.optional(v.string()),
auto_parameters: v.optional(v.record(v.string(), v.unknown())),
usage: v.optional(v.object({ credits: v.number() }))
});
const normalize_tavily_date = (date) => {
if (/^\d{4}$/.test(date)) return `${date}-01-01`;
if (/^\d{4}-\d{2}$/.test(date)) return `${date}-01`;
return date;
};
const tavily_country_aliases = {
uk: "united kingdom",
us: "united states",
usa: "united states"
};
const normalize_tavily_country = (location) => {
const normalized = location.toLowerCase().replace(/-/g, " ");
return tavily_country_aliases[normalized] ?? normalized;
};
var TavilySearchProvider = class {
name = "tavily";
description = "Search the web using Tavily Search API. Best for factual queries requiring reliable sources and citations. Supports domain filtering through API parameters (include_domains/exclude_domains). Provides high-quality results for technical, scientific, and academic topics. Use when you need verified information with strong citation support.";
async search(params) {
const api_key = validate_api_key(config.search.tavily.api_key, this.name);
const parsed_query = parse_search_operators(params.query);
const search_params = apply_search_operators(parsed_query);
const search_request = async () => {
try {
const include_domains = [...params.include_domains ?? [], ...search_params.include_domains ?? []];
const exclude_domains = [...params.exclude_domains ?? [], ...search_params.exclude_domains ?? []];
const request_body = {
query: sanitize_query(search_params.query),
max_results: params.limit ?? 5,
include_domains: include_domains.length > 0 ? include_domains : [],
exclude_domains: exclude_domains.length > 0 ? exclude_domains : [],
search_depth: params.search_depth ?? "basic",
topic: params.topic ?? "general",
include_favicon: true,
include_usage: true
};
if (params.time_range) request_body.time_range = params.time_range;
if (params.safe_search !== void 0) request_body.safe_search = params.safe_search;
if (params.include_raw_content !== void 0) request_body.include_raw_content = params.include_raw_content;
if (params.auto_parameters !== void 0) request_body.auto_parameters = params.auto_parameters;
if (search_params.date_after) request_body.start_date = normalize_tavily_date(search_params.date_after);
if (search_params.date_before) request_body.end_date = normalize_tavily_date(search_params.date_before);
if (search_params.exact_phrases && search_params.exact_phrases.length > 0) {
request_body.exact_match = true;
const exact_query_parts = search_params.exact_phrases.map((phrase) => `"${phrase}"`);
request_body.query = `${request_body.query} ${exact_query_parts.join(" ")}`.trim();
}
if (search_params.location) request_body.country = normalize_tavily_country(search_params.location);
const raw_data = await http_json(this.name, `${config.search.tavily.base_url}/search`, {
method: "POST",
headers: {
Authorization: `Bearer ${api_key}`,
"Content-Type": "application/json"
},
body: JSON.stringify(request_body)
});
const data = parse_provider_response(this.name, tavily_search_response_schema, raw_data);
return data.results.map((result) => ({
title: result.title,
url: result.url,
snippet: result.content,
score: result.score,
source_provider: this.name,
metadata: {
...params.include_raw_content && result.raw_content ? { raw_content: result.raw_content } : {},
...result.favicon ? { favicon: result.favicon } : {},
...data.request_id ? { request_id: data.request_id } : {},
...data.response_time !== void 0 ? { response_time: data.response_time } : {},
...data.auto_parameters ? { auto_parameters: data.auto_parameters } : {},
...data.usage ? { usage: data.usage } : {}
}
}));
} catch (error) {
handle_provider_error(error, this.name, "fetch search results");
}
};
return retry_with_backoff(search_request);
}
};
//#endregion
//#region src/server/provider-definitions.ts
const make_processing_provider_key = (provider, mode) => `${provider}:${mode}`;
const web_search_provider_definitions = [
{
id: "tavily",
name: "tavily",
category: "search",
api_key_name: "TAVILY_API_KEY",
tools: ["web_search"],
capabilities: [
"web_search",
"domain_filters",
"operator_translation"
],
api_key: config.search.tavily.api_key,
create: () => new TavilySearchProvider()
},
{
id: "brave",
name: "brave",
category: "search",
api_key_name: "BRAVE_API_KEY",
tools: ["web_search"],
capabilities: [
"web_search",
"domain_filters",
"operator_passthrough"
],
api_key: config.search.brave.api_key,
create: () => new BraveSearchProvider()
},
{
id: "kagi",
name: "kagi",
category: "search",
api_key_name: "KAGI_API_KEY",
tools: ["web_search"],
capabilities: [
"web_search",
"domain_filters",
"operator_passthrough"
],
api_key: config.search.kagi.api_key,
create: () => new KagiSearchProvider()
},
{
id: "exa",
name: "exa",
category: "search",
api_key_name: "EXA_API_KEY",
tools: ["web_search"],
capabilities: [
"web_search",
"domain_filters",
"semantic_search"
],
api_key: config.search.exa.api_key,
create: () => new ExaSearchProvider()
},
{
id: "kagi_enrichment",
name: "kagi_enrichment",
category: "search",
api_key_name: "KAGI_API_KEY",
tools: ["web_search"],
capabilities: ["specialized_indexes", "web_enrichment"],
api_key: config.enhancement.kagi_enrichment.api_key,
create: () => new KagiEnrichmentSearchProvider()
}
];
const ai_search_provider_definitions = [
{
id: "kagi_fastgpt",
name: "kagi_fastgpt",
category: "ai_response",
api_key_name: "KAGI_API_KEY",
tools: ["ai_search"],
capabilities: ["answer_generation", "citations"],
api_key: config.ai_response.kagi_fastgpt.api_key,
create: () => new KagiFastGPTProvider()
},
{
id: "exa_answer",
name: "exa_answer",
category: "ai_response",
api_key_name: "EXA_API_KEY",
tools: ["ai_search"],
capabilities: ["answer_generation", "semantic_search"],
api_key: config.ai_response.exa_answer.api_key,
create: () => new ExaAnswerProvider()
},
{
id: "linkup",
name: "linkup",
category: "ai_response",
api_key_name: "LINKUP_API_KEY",
tools: ["ai_search"],
capabilities: ["answer_generation", "citations"],
api_key: config.ai_response.linkup.api_key,
create: () => new LinkupProvider()
},
{
id: "tavily_research",
name: "tavily_research",
category: "ai_response",
api_key_name: "TAVILY_API_KEY",
tools: ["ai_search"],
capabilities: [
"deep_research",
"answer_generation",
"citations"
],
api_key: config.ai_response.tavily_research.api_key,
create: () => new TavilyResearchProvider()
}
];
const github_provider_definitions = [{
id: "github",
name: "github",
category: "search",
api_key_name: "GITHUB_API_KEY",
tools: ["github_search"],
modes: [
"code",
"repositories",
"users"
],
capabilities: [
"code_search",
"repository_search",
"user_search"
],
api_key: config.search.github.api_key,
create: () => new GitHubSearchProvider()
}];
const web_extract_provider_definitions = [
{
id: make_processing_provider_key("tavily", "extract"),
name: "tavily",
category: "processing",
api_key: config.processing.tavily_extract.api_key,
api_key_name: "TAVILY_API_KEY",
tools: ["web_extract"],
modes: ["extract"],
capabilities: ["content_extraction", "raw_contents"],
default_mode: true,
create: () => new TavilyExtractProvider()
},
{
id: make_processing_provider_key("tavily", "crawl"),
name: "tavily",
category: "processing",
api_key: config.processing.tavily_crawl.api_key,
api_key_name: "TAVILY_API_KEY",
tools: ["web_extract"],
modes: ["crawl"],
capabilities: [
"crawling",
"content_extraction",
"raw_contents"
],
create: () => new TavilyCrawlProvider()
},
{
id: make_processing_provider_key("tavily", "map"),
name: "tavily",
category: "processing",
api_key: config.processing.tavily_map.api_key,
api_key_name: "TAVILY_API_KEY",
tools: ["web_extract"],
modes: ["map"],
capabilities: ["site_mapping"],
create: () => new TavilyMapProvider()
},
{
id: make_processing_provider_key("kagi", "summarize"),
name: "kagi",
category: "processing",
api_key: config.processing.kagi_summarizer.api_key,
api_key_name: "KAGI_API_KEY",
tools: ["web_extract"],
modes: ["summarize"],
capabilities: ["summarization"],
default_mode: true,
create: () => new KagiSummarizerProvider()
},
{
id: make_processing_provider_key("firecrawl", "scrape"),
name: "firecrawl",
category: "processing",
api_key: config.processing.firecrawl_scrape.api_key,
api_key_name: "FIRECRAWL_API_KEY",
tools: ["web_extract"],
modes: ["scrape"],
capabilities: ["scraping"],
default_mode: true,
create: () => new FirecrawlScrapeProvider()
},
{
id: make_processing_provider_key("firecrawl", "crawl"),
name: "firecrawl",
category: "processing",
api_key: config.processing.firecrawl_crawl.api_key,
api_key_name: "FIRECRAWL_API_KEY",
tools: ["web_extract"],
modes: ["crawl"],
capabilities: ["crawling"],
create: () => new FirecrawlCrawlProvider()
},
{
id: make_processing_provider_key("firecrawl", "map"),
name: "firecrawl",
category: "processing",
api_key: config.processing.firecrawl_map.api_key,
api_key_name: "FIRECRAWL_API_KEY",
tools: ["web_extract"],
modes: ["map"],
capabilities: ["site_mapping"],
create: () => new FirecrawlMapProvider()
},
{
id: make_processing_provider_key("firecrawl", "extract"),
name: "firecrawl",
category: "processing",
api_key: config.processing.firecrawl_extract.api_key,
api_key_name: "FIRECRAWL_API_KEY",
tools: ["web_extract"],
modes: ["extract"],
capabilities: ["structured_extraction"],
create: () => new FirecrawlExtractProvider()
},
{
id: make_processing_provider_key("firecrawl", "actions"),
name: "firecrawl",
category: "processing",
api_key: config.processing.firecrawl_actions.api_key,
api_key_name: "FIRECRAWL_API_KEY",
tools: ["web_extract"],
modes: ["actions"],
capabilities: ["browser_actions"],
create: () => new FirecrawlActionsProvider()
},
{
id: make_processing_provider_key("exa", "contents"),
name: "exa",
category: "processing",
api_key: config.processing.exa_contents.api_key,
api_key_name: "EXA_API_KEY",
tools: ["web_extract"],
modes: ["contents"],
capabilities: ["content_retrieval"],
default_mode: true,
create: () => new ExaContentsProvider()
},
{
id: make_processing_provider_key("exa", "similar"),
name: "exa",
category: "processing",
api_key: config.processing.exa_similar.api_key,
api_key_name: "EXA_API_KEY",
tools: ["web_extract"],
modes: ["similar"],
capabilities: ["similar_pages"],
create: () => new ExaSimilarProvider()
}
];
const get_default_web_extract_mode = (provider) => web_extract_provider_definitions.find((definition) => definition.name === provider && definition.default_mode)?.modes[0];
const get_valid_web_extract_modes = (provider) => web_extract_provider_definitions.filter((definition) => definition.name === provider).map((definition) => definition.modes[0]);
//#endregion
//#region src/server/provider-registry.ts
var ProviderRegistry = class {
providers = /* @__PURE__ */ new Map();
statuses = /* @__PURE__ */ new Map();
missing_api_key_names = /* @__PURE__ */ new Set();
clear() {
this.providers.clear();
this.statuses.clear();
this.missing_api_key_names.clear();
}
register(definition) {
const api_key_name = definition.api_key_name ?? definition.name;
const base_status = {
id: definition.id,
name: definition.name,
category: definition.category,
api_key_name,
description: definition.description,
tools: definition.tools ?? [],
modes: definition.modes ?? [],
capabilities: definition.capabilities ?? []
};
if (!definition.api_key || definition.api_key.trim() === "") {
if (!this.missing_api_key_names.has(api_key_name)) {
is_api_key_valid(definition.api_key, api_key_name);
this.missing_api_key_names.add(api_key_name);
}
this.statuses.set(definition.id, {
...base_status,
status: "unavailable",
unavailable_reason: "missing_api_key"
});
return;
}
if (!is_api_key_valid(definition.api_key, api_key_name)) {
this.statuses.set(definition.id, {
...base_status,
status: "unavailable",
unavailable_reason: "missing_api_key"
});
return;
}
const instance = definition.create();
this.providers.set(definition.id, {
...base_status,
instance,
description: definition.description ?? instance.description
});
this.statuses.set(definition.id, {
...base_status,
status: "available",
description: definition.description ?? instance.description
});
}
register_all(definitions) {
for (const definition of definitions) this.register(definition);
}
get(id) {
return this.providers.get(id)?.instance;
}
require(id, tool_name, message = `Provider "${id}" is not available. Available: ${this.ids().join(", ")}`) {
const provider = this.get(id);
if (!provider) throw new ProviderError("INVALID_INPUT", message, tool_name);
return provider;
}
ids() {
return Array.from(this.providers.keys());
}
names() {
return Array.from(new Set(Array.from(this.providers.values()).map((provider) => provider.name)));
}
entries() {
return Array.from(this.providers.values());
}
status_entries() {
return Array.from(this.statuses.values());
}
get size() {
return this.providers.size;
}
};
//#endregion
//#region src/server/tools/responses.ts
const create_json_tool_response = (payload) => ({ content: [{
type: "text",
text: JSON.stringify(payload, null, 2)
}] });
const create_error_tool_response = (error) => ({
...create_json_tool_response(create_error_response(error)),
isError: true
});
const handle_tool_result = async (tool_name, result, options = {}) => {
try {
return create_json_tool_response(handle_large_result(await result(), tool_name, { mode: options.large_result_mode }));
} catch (error) {
return create_error_tool_response(error);
}
};
//#endregion
//#region src/server/tools/schemas.ts
const DOMAIN_PATTERN = /^(?:\*\.)?(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,63}$/;
const query_schema = v.pipe(v.string(), v.minLength(1, "Query cannot be empty"), v.regex(/\S/, "Query cannot be empty"), v.description("Search query"));
const limit_schema = v.optional(v.pipe(v.number(), v.integer("Limit must be an integer"), v.minValue(1, "Limit must be at least 1"), v.maxValue(50, "Limit must be at most 50"), v.description("Maximum number of results (default: 10)")));
const search_depth_schema = v.optional(v.pipe(v.picklist([
"basic",
"advanced",
"fast",
"ultra-fast"
]), v.description("Search depth. Providers may use this to balance speed, relevance, and cost.")));
const search_topic_schema = v.optional(v.pipe(v.picklist([
"general",
"news",
"finance"
]), v.description("Search topic category.")));
const search_time_range_schema = v.optional(v.pipe(v.picklist([
"day",
"week",
"month",
"year"
]), v.description("Only return results from this recent time range.")));
const safe_search_schema = v.optional(v.pipe(v.boolean(), v.description("Enable provider safe-search filtering.")));
const auto_parameters_schema = v.optional(v.pipe(v.boolean(), v.description("Let supported providers select search settings from the query. This can change cost.")));
const include_raw_content_schema = v.optional(v.pipe(v.boolean(), v.description("Include full page content when the selected provider supports it.")));
const large_result_mode_schema = v.optional(v.pipe(v.picklist(["inline", "file"]), v.description("How to handle oversized responses for this request. Use inline for remote/container transports; file is local shared-filesystem behavior. Defaults to OMNISEARCH_LARGE_RESULT_MODE or file.")));
const extraction_query_schema = v.optional(v.pipe(v.string(), v.minLength(1, "Extraction query cannot be empty"), v.regex(/\S/, "Extraction query cannot be empty"), v.description("Focus extracted content on information relevant to this query.")));
const chunks_per_source_schema = v.optional(v.pipe(v.number(), v.integer("Chunks per source must be an integer"), v.minValue(1, "Chunks per source must be at least 1"), v.maxValue(5, "Chunks per source must be at most 5"), v.description("Maximum relevant content chunks per source when a query is provided.")));
const extraction_format_schema = v.optional(v.pipe(v.picklist(["markdown", "text"]), v.description("Extracted page format (default: markdown).")));
const include_raw_contents_schema = v.optional(v.pipe(v.boolean(), v.description("Whether extraction responses should include per-URL raw_contents alongside combined content (default: true).")));
const domain_schema = v.pipe(v.string(), v.regex(DOMAIN_PATTERN, "Domain must be a hostname, not a URL"));
const include_domains_schema = v.optional(v.pipe(v.array(domain_schema), v.maxLength(20, "Use at most 20 included domains"), v.description("Only return results from these domains")));
const exclude_domains_schema = v.optional(v.pipe(v.array(domain_schema), v.maxLength(20, "Use at most 20 excluded domains"), v.description("Exclude results from these domains")));
const http_url_schema = v.pipe(v.string(), v.url("URL must be valid"), v.regex(/^https?:\/\//, "URL protocol must be http or https"));
const url_or_urls_schema = v.pipe(v.union([http_url_schema, v.pipe(v.array(http_url_schema), v.minLength(1, "Provide at least one URL"), v.maxLength(10, "Use at most 10 URLs per extraction"))]), v.description("URL or array of URLs to process"));
//#endregion
//#region src/server/tools/ai-search.ts
const providers$3 = new ProviderRegistry();
const initialize_ai_search = () => {
providers$3.clear();
providers$3.register_all(ai_search_provider_definitions);
return providers$3.size > 0;
};
const get_available_providers$2 = () => providers$3.names();
const get_provider_status_entries$3 = () => providers$3.status_entries();
const register_ai_search = (server) => {
if (providers$3.size === 0) return;
const provider_names = providers$3.ids();
server.tool({
name: "ai_search",
description: "Get AI-powered answers with citations and reasoning. Use when you need synthesized answers rather than raw search results. Providers: kagi_fastgpt (fast answers), exa_answer (semantic AI), linkup (deep agentic search), tavily_research (asynchronous multi-search reports; resubmit its research_id to retrieve results).",
annotations: {
readOnlyHint: true,
destructiveHint: false,
idempotentHint: true,
openWorldHint: true
},
schema: v.object({
query: query_schema,
provider: v.pipe(v.picklist(provider_names), v.description("AI search provider to use")),
limit: limit_schema,
research_id: v.optional(v.pipe(v.string(), v.minLength(1), v.description("Existing asynchronous research task ID to retrieve. Supported by Tavily Research."))),
large_result_mode: large_result_mode_schema
})
}, async ({ query, provider, limit, research_id, large_result_mode }) => handle_tool_result("ai_search", async () => {
return providers$3.require(provider, "ai_search").search({
query,
limit,
research_id
});
}, { large_result_mode }));
};
//#endregion
//#region src/server/tools/github-search.ts
const providers$2 = new ProviderRegistry();
const initialize_github_search = () => {
providers$2.clear();
providers$2.register_all(github_provider_definitions);
return providers$2.size > 0;
};
const get_available = () => providers$2.names();
const get_provider_status_entries$2 = () => providers$2.status_entries();
const register_github_search = (server) => {
if (providers$2.size === 0) return;
server.tool({
name: "github_search",
description: "Search GitHub for code, repositories, or users. Use when you need to find code examples, open source projects, or developers. Supports advanced syntax: filename:, path:, repo:, user:, language:, in:file.",
annotations: {
readOnlyHint: true,
destructiveHint: false,
idempotentHint: true,
openWorldHint: true
},
schema: v.object({
query: query_schema,
search_type: v.optional(v.pipe(v.picklist([
"code",
"repositories",
"users"
]), v.description("What to search for (default: code)"))),
limit: limit_schema,
large_result_mode: large_result_mode_schema,
sort: v.optional(v.pipe(v.picklist([
"stars",
"forks",
"updated"
]), v.description("Sort order (repositories only)")))
})
}, async ({ query, search_type = "code", limit, large_result_mode, sort }) => handle_tool_result("github_search", async () => {
const selected = providers$2.require("github", "github_search");
switch (search_type) {
case "code": return selected.search_code({
query,
limit
});
case "repositories": return selected.search_repositories({
query,
limit,
sort
});
case "users": return selected.search_users({
query,
limit
});
}
}, { large_result_mode }));
};
//#endregion
//#region src/server/tools/web-extract.ts
const providers$1 = new ProviderRegistry();
const initialize_web_extract = () => {
providers$1.clear();
providers$1.register_all(web_extract_provider_definitions);
return providers$1.size > 0;
};
const get_available_providers$1 = () => providers$1.names();
const get_provider_status_entries$1 = () => providers$1.status_entries();
const web_extract_modes = Array.from(new Set(web_extract_provider_definitions.map((definition) => definition.modes[0])));
const register_web_extract = (server) => {
if (providers$1.size === 0) return;
const available = get_available_providers$1();
server.tool({
name: "web_extract",
description: "Extract, process, or summarize web content from URLs. Use when you need to read page content, summarize articles, crawl sites, or extract structured data. Providers: tavily (content extraction), kagi (summarization of pages/videos/podcasts), firecrawl (scraping/crawling/mapping/structured extraction/interactive), exa (content retrieval/similar pages).",
annotations: {
readOnlyHint: true,
destructiveHint: false,
idempotentHint: true,
openWorldHint: true
},
schema: v.object({
url: url_or_urls_schema,
provider: v.pipe(v.picklist(available), v.description("Processing provider to use")),
mode: v.optional(v.pipe(v.picklist(web_extract_modes), v.description("Processing mode. Firecrawl: scrape/crawl/map/extract/actions. Exa: contents/similar. Tavily: extract/crawl/map. Kagi: summarize. Defaults to provider default."))),
extract_depth: v.optional(v.pipe(v.picklist(["basic", "advanced"]), v.description("Extraction depth (default: basic)"))),
query: extraction_query_schema,
chunks_per_source: chunks_per_source_schema,
format: extraction_format_schema,
large_result_mode: large_result_mode_schema,
include_raw_contents: include_raw_contents_schema
})
}, async ({ url, provider, mode, extract_depth, query, chunks_per_source, format, large_result_mode, include_raw_contents = true }) => handle_tool_result("web_extract", async () => {
if (chunks_per_source !== void 0 && !query) throw new ProviderError("INVALID_INPUT", "query is required when chunks_per_source is provided", "web_extract");
const provider_name = provider;
const resolved_mode = mode || get_default_web_extract_mode(provider_name);
const allowed = get_valid_web_extract_modes(provider_name);
if (!resolved_mode || !allowed.includes(resolved_mode)) throw new ProviderError("INVALID_INPUT", `Mode "${resolved_mode}" is not valid for provider "${provider}". Valid modes: ${allowed.join(", ")}`, "web_extract");
const key = make_processing_provider_key(provider, resolved_mode);
const result = await providers$1.require(key, "web_extract", `Provider "${provider}" with mode "${resolved_mode}" is not available. Available modes for configured providers: ${allowed.join(", ") || "none"}.`).process_content(url, extract_depth, {
query,
chunks_per_source,
format
});
return include_raw_contents ? result : omit_raw_contents(result);
}, { large_result_mode }));
};
//#endregion
//#region src/server/tools/web-search.ts
const providers = new ProviderRegistry();
const initialize_web_search = () => {
providers.clear();
providers.register_all(web_search_provider_definitions);
return providers.size > 0;
};
const get_available_providers = () => providers.names();
const get_provider_status_entries = () => providers.status_entries();
const register_web_search = (server) => {
if (providers.size === 0) return;
const provider_names = providers.ids();
server.tool({
name: "web_search",
description: "Search the web for information. Use when you need to find web pages, articles, or data. Providers: tavily (factual/citations and search controls), brave (privacy/operators), kagi (quality/operators), exa (AI-semantic), kagi_enrichment (specialized indexes). Search depth, topic, time range, safe search, raw content, and automatic parameters apply when supported by the provider.",
annotations: {
readOnlyHint: true,
destructiveHint: false,
idempotentHint: true,
openWorldHint: true
},
schema: v.object({
query: query_schema,
provider: v.pipe(v.picklist(provider_names), v.description("Search provider to use")),
limit: limit_schema,
include_domains: include_domains_schema,
exclude_domains: exclude_domains_schema,
search_depth: search_depth_schema,
topic: search_topic_schema,
time_range: search_time_range_schema,
safe_search: safe_search_schema,
include_raw_content: include_raw_content_schema,
auto_parameters: auto_parameters_schema,
large_result_mode: large_result_mode_schema
})
}, async ({ query, provider, limit, include_domains, exclude_domains, search_depth, topic, time_range, safe_search, include_raw_content, auto_parameters, large_result_mode }) => handle_tool_result("web_search", async () => {
return providers.require(provider, "web_search").search({
query,
limit,
include_domains,
exclude_domains,
search_depth,
topic,
time_range,
safe_search,
include_raw_content,
auto_parameters
});
}, { large_result_mode }));
};
//#endregion
//#region src/server/tools/index.ts
const available_providers = {
search: /* @__PURE__ */ new Set(),
ai_response: /* @__PURE__ */ new Set(),
processing: /* @__PURE__ */ new Set()
};
const provider_status_entries = [];
const reset_provider_tracking = () => {
available_providers.search.clear();
available_providers.ai_response.clear();
available_providers.processing.clear();
provider_status_entries.length = 0;
};
const initialize_providers = () => {
reset_provider_tracking();
if (initialize_web_search()) for (const p of get_available_providers()) available_providers.search.add(p);
provider_status_entries.push(...get_provider_status_entries());
if (initialize_github_search()) for (const p of get_available()) available_providers.search.add(p);
provider_status_entries.push(...get_provider_status_entries$2());
if (initialize_ai_search()) for (const p of get_available_providers$2()) available_providers.ai_response.add(p);
provider_status_entries.push(...get_provider_status_entries$3());
if (initialize_web_extract()) for (const p of get_available_providers$1()) available_providers.processing.add(p);
provider_status_entries.push(...get_provider_status_entries$1());
console.error("Available providers:");
if (available_providers.search.size > 0) console.error(`- Search: ${Array.from(available_providers.search).join(", ")}`);
else console.error("- Search: None available (missing API keys)");
if (available_providers.ai_response.size > 0) console.error(`- AI Response: ${Array.from(available_providers.ai_response).join(", ")}`);
else console.error("- AI Response: None available (missing API keys)");
if (available_providers.processing.size > 0) console.error(`- Processing: ${Array.from(available_providers.processing).join(", ")}`);
else console.error("- Processing: None available (missing API keys)");
};
const register_tools = (server) => {
register_web_search(server);
register_github_search(server);
register_ai_search(server);
register_web_extract(server);
};
//#endregion
//#region src/server/handlers.ts
const categories = [
"search",
"ai_response",
"processing"
];
const unique = (values) => Array.from(new Set(values)).sort();
const grouped_provider_status = () => {
const grouped = {
search: [],
ai_response: [],
processing: []
};
for (const provider of provider_status_entries) grouped[provider.category].push(provider);
return grouped;
};
const aggregate_provider_info = (provider_name, category) => {
const entries = provider_status_entries.filter((provider) => (provider.id === provider_name || provider.name === provider_name) && (!category || provider.category === category));
if (entries.length === 0) return void 0;
return {
name: provider_name,
status: entries.some((provider) => provider.status === "available") ? "available" : "unavailable",
categories: unique(entries.map((provider) => provider.category)),
tools: unique(entries.flatMap((provider) => provider.tools)),
modes: unique(entries.flatMap((provider) => provider.modes)),
capabilities: unique(entries.flatMap((provider) => provider.capabilities)),
providers: entries.map((provider) => ({
id: provider.id,
name: provider.name,
category: provider.category,
status: provider.status,
api_key_name: provider.api_key_name,
description: provider.description,
tools: provider.tools,
modes: provider.modes,
capabilities: provider.capabilities,
unavailable_reason: provider.unavailable_reason
}))
};
};
const setup_handlers = (server) => {
server.resource({
name: "provider-status",
description: "Current status of all configured providers",
uri: "omnisearch://providers/status"
}, async () => {
const providers = grouped_provider_status();
const available_count = Object.fromEntries(categories.map((category) => [category, providers[category].filter((provider) => provider.status === "available").length]));
const unavailable_count = Object.fromEntries(categories.map((category) => [category, providers[category].filter((provider) => provider.status === "unavailable").length]));
const total = provider_status_entries.length;
const available_total = categories.reduce((sum, category) => sum + available_count[category], 0);
return { contents: [{
uri: "omnisearch://providers/status",
mimeType: "application/json",
text: JSON.stringify({
status: available_total === 0 ? "unavailable" : available_total === total ? "operational" : "degraded",
providers,
available_count: {
...available_count,
total: available_total
},
unavailable_count: {
...unavailable_count,
total: total - available_total
}
}, null, 2)
}] };
});
server.resource({
name: "provider-info",
description: "Information about a specific configured provider",
uri: "omnisearch://providers/{provider}/info"
}, async (uri) => {
const providerMatch = uri.match(/^omnisearch:\/\/(providers|search|ai_response|processing)\/([^/]+)\/info$/);
if (providerMatch) {
const [, scope, providerName] = providerMatch;
const providerInfo = aggregate_provider_info(providerName, scope === "providers" ? void 0 : scope);
if (!providerInfo) throw new Error(`Unknown provider: ${providerName}`);
return { contents: [{
uri,
mimeType: "application/json",
text: JSON.stringify(providerInfo, null, 2)
}] };
}
throw new Error(`Unknown resource URI: ${uri}`);
});
};
//#endregion
//#region src/mcp-server.ts
/** Create a fully registered MCP server for a transport. */
function create_mcp_server(metadata) {
const server = new McpServer(metadata, {
adapter: new ValibotJsonSchemaAdapter(),
capabilities: {
tools: { listChanged: true },
resources: { listChanged: true }
}
});
initialize_providers();
register_tools(server);
setup_handlers(server);
return server;
}
//#endregion
//#region src/index.ts
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const { name, version } = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf8"));
validate_config();
const server = create_mcp_server({
name,
version,
description: "MCP server for integrating Omnisearch with LLMs"
});
process.on("SIGINT", () => {
process.exit(0);
});
new StdioTransport(server).listen();
console.error("Omnisearch MCP server running on stdio");
//#endregion
export {};
//# sourceMappingURL=index.js.map