markov-exa-mcp-server
Version:
A Model Context Protocol server with Exa for web search, academic paper search, and Twitter/X.com search. Provides real-time web searches with configurable tool selection, allowing users to enable or disable specific search capabilities. Supports customiz
579 lines (568 loc) • 30.5 kB
JavaScript
import { z } from "zod";
import axios from "axios";
import { createRequestLogger } from "../utils/logger.js";
import * as fs from "fs/promises";
import * as path from "path";
// Store for OpenAI deep research task results
const deepResearchCache = new Map();
// Helper function to clear old cache entries (keep for 2 hours due to longer task times)
function cleanupCache() {
const twoHoursAgo = Date.now() - 2 * 60 * 60 * 1000;
for (const [key, value] of deepResearchCache.entries()) {
if (!value._cachedAt || value._cachedAt < twoHoursAgo) {
deepResearchCache.delete(key);
}
}
}
const OPENAI_API_BASE = "https://api.openai.com/v1";
const REQUEST_TIMEOUT = 60000; // 1 minute for create requests
const STATUS_TIMEOUT = 30000; // 30 seconds for status checks
// Helper function to enhance user queries with prompt rewriting (following ChatGPT Deep Research pattern)
async function enhanceQueryForDeepResearch(userQuery, apiKey) {
const promptRewritingInstructions = `
You will be given a research task by a user. Your job is to produce a set of detailed instructions for a deep research model that will complete the task. Do NOT complete the task yourself, just provide comprehensive instructions on how to complete it.
GUIDELINES:
1. **Maximize Specificity and Detail**
- Include all known user preferences and explicitly list key attributes or dimensions to consider.
- It is of utmost importance that all details from the user are included in the instructions.
2. **Fill in Unstated But Necessary Dimensions as Open-Ended**
- If certain attributes are essential for a meaningful output but the user has not provided them, explicitly state that they are open-ended or default to no specific constraint.
3. **Avoid Unwarranted Assumptions**
- If the user has not provided a particular detail, do not invent one.
- Instead, state the lack of specification and guide the researcher to treat it as flexible or accept all possible options.
4. **Use the First Person**
- Phrase the request from the perspective of the user.
5. **Structure and Format Requirements**
- If you determine that including tables, charts, or structured data will help illustrate, organize, or enhance the information in the research output, you must explicitly request that the researcher provide them.
- If the user is asking for content that would be best returned in a structured format (e.g. a report, plan, etc.), ask the researcher to format as a report with appropriate headers and formatting that ensures clarity and structure.
6. **Source Quality Guidelines**
- Prioritize reliable, up-to-date sources: peer-reviewed research, authoritative organizations, regulatory agencies, or official company reports.
- For academic or scientific queries, prefer linking directly to the original paper or official journal publication rather than survey papers or secondary summaries.
- For product and business research, prefer linking directly to official or primary websites rather than aggregator sites or SEO-heavy blogs.
7. **Evidence and Citation Requirements (MANDATORY)**
- ALWAYS request specific figures, trends, statistics, and measurable outcomes in the research output.
- ALWAYS ask for inline citations and source metadata to be included for every claim.
- ALWAYS prioritize reliable, up-to-date sources in the research instructions.
- Ensure that each section supports data-backed reasoning rather than generalities.
8. **Research Methodology**
- Guide the researcher to be analytical and avoid generalities.
- Request that findings be grounded in verifiable evidence.
- Ask for multiple perspectives when analyzing complex topics.
IMPORTANT: Transform the user's query into comprehensive research instructions that will minimize hallucination and maximize factual accuracy. The enhanced prompt should be detailed enough that a research model can execute it without making unwarranted assumptions.
MANDATORY TEMPLATE - ALWAYS include these exact instructions in your output:
"In your research response, you MUST:
- Include specific figures, trends, statistics, and measurable outcomes for every claim made
- Prioritize reliable, up-to-date sources: peer-reviewed research, authoritative organizations, regulatory agencies, or official company reports
- Include inline citations and return all source metadata for every fact, statistic, or claim
- Avoid generalizations and ensure every statement is backed by verifiable evidence
- When data is unavailable, explicitly state this rather than making assumptions"
`;
try {
const response = await axios.post(`${OPENAI_API_BASE}/responses`, {
model: "gpt-4.1",
input: userQuery,
instructions: promptRewritingInstructions
}, {
headers: {
"Authorization": `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
timeout: REQUEST_TIMEOUT,
});
return response.data.output_text || userQuery; // Fallback to original query if enhancement fails
}
catch (error) {
console.warn('Failed to enhance query, using original:', error);
return userQuery; // Fallback to original query
}
}
export function registerOpenAIDeepResearchTools(server) {
// Tool to create an OpenAI deep research task
server.tool("openai_deep_research_create", "Create a comprehensive deep research task using OpenAI's o3-deep-research or o4-mini-deep-research models. Executes actual research with web search and returns detailed findings. Your query is enhanced with gpt-4.1 to minimize hallucinations before research begins.", {
query: z.string().describe("Your actual research question or task. This is what will be researched. Example: 'Find all independent accounting firms in Belgium with 20+ employees'"),
model: z.enum(["o3-deep-research", "o4-mini-deep-research"]).default("o4-mini-deep-research").describe("Deep research model to use (default: o4-mini-deep-research)"),
enableWebSearch: z.boolean().default(true).describe("Enable web search capabilities (default: true)"),
enableCodeInterpreter: z.boolean().default(true).describe("Enable code interpreter for data analysis (default: true)")
}, async ({ query, model, enableWebSearch, enableCodeInterpreter }) => {
const requestId = `openai_deep_research_create-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`;
const logger = createRequestLogger(requestId, 'openai_deep_research_create');
try {
const apiKey = process.env.OPENAI_API_KEY || process.env.openaiApiKey;
if (!apiKey) {
logger.error('No OpenAI API key found');
return {
content: [{
type: "text",
text: "OPENAI_API_KEY environment variable is not set"
}],
isError: true
};
}
// Use the provided query
const finalQuery = query;
logger.log(`Processing research query: ${finalQuery.substring(0, 100)}...`);
// Build tools array based on enabled features
const tools = [];
if (enableWebSearch) {
tools.push({ type: "web_search_preview" });
}
if (enableCodeInterpreter) {
tools.push({
type: "code_interpreter",
container: { type: "auto" }
});
}
if (tools.length === 0) {
return {
content: [{
type: "text",
text: "At least one tool (web search or code interpreter) must be enabled for deep research"
}],
isError: true
};
}
// Step 1: Enhance the user query using gpt-4.1
logger.log('Enhancing user query with prompt rewriting...');
const enhancedQuery = await enhanceQueryForDeepResearch(finalQuery, apiKey);
logger.log(`Original query: ${finalQuery.substring(0, 100)}...`);
logger.log(`Enhanced query: ${enhancedQuery.substring(0, 200)}...`);
const requestBody = {
model,
input: enhancedQuery, // Use enhanced query instead of raw user input
background: true,
tools
};
logger.log(`Sending POST request to ${OPENAI_API_BASE}/responses`);
logger.log(`Request body: ${JSON.stringify(requestBody, null, 2)}`);
const response = await axios.post(`${OPENAI_API_BASE}/responses`, requestBody, {
headers: {
"Authorization": `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
timeout: REQUEST_TIMEOUT,
});
logger.log(`Response status: ${response.status}`);
logger.log(`Response data: ${JSON.stringify(response.data, null, 2)}`);
logger.complete();
return {
content: [{
type: "text",
text: JSON.stringify({
taskId: response.data.id,
status: response.data.status,
model: model,
message: `Deep research task created successfully with enhanced prompting. Task ID: ${response.data.id}. Your query was optimized using gpt-4.1 to minimize hallucinations. This is a background task that may take several minutes to complete. Use 'openai_deep_research_check_status' to monitor progress.`,
enabledTools: {
webSearch: enableWebSearch,
codeInterpreter: enableCodeInterpreter
}
}, null, 2)
}]
};
}
catch (error) {
logger.error(error);
if (axios.isAxiosError(error)) {
const errorMessage = error.response?.data?.error?.message || error.response?.data?.message || error.message;
return {
content: [{
type: "text",
text: `Failed to create OpenAI deep research task: ${errorMessage}`
}],
isError: true
};
}
return {
content: [{
type: "text",
text: `Failed to create OpenAI deep research task: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true
};
}
});
// Tool to check deep research task status
server.tool("openai_deep_research_check_status", "Check the status of an OpenAI deep research background task. Returns status and partial results if available.", {
taskId: z.string().describe("The task ID returned from creating a deep research task")
}, async ({ taskId }) => {
const requestId = `openai_deep_research_status-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`;
const logger = createRequestLogger(requestId, 'openai_deep_research_check_status');
try {
const apiKey = process.env.OPENAI_API_KEY || process.env.openaiApiKey;
if (!apiKey) {
logger.error('No OpenAI API key found');
return {
content: [{
type: "text",
text: "OPENAI_API_KEY environment variable is not set"
}],
isError: true
};
}
const url = `${OPENAI_API_BASE}/responses/${taskId}`;
logger.log(`Sending GET request to ${url}`);
const response = await axios.get(url, {
headers: {
"Authorization": `Bearer ${apiKey}`,
},
timeout: STATUS_TIMEOUT,
});
logger.log(`Response status: ${response.status}`);
const responseSize = JSON.stringify(response.data).length;
logger.log(`Response size: ${responseSize} characters`);
logger.complete();
// Cache the response if completed
if (response.data.status === 'completed') {
cleanupCache();
response.data._cachedAt = Date.now();
deepResearchCache.set(taskId, response.data);
}
if (response.data.status === 'failed') {
return {
content: [{
type: "text",
text: JSON.stringify({
taskId: response.data.id,
status: response.data.status,
error: response.data.error?.message || 'Deep research task failed with no error message'
}, null, 2)
}],
isError: true
};
}
// For large completed responses, provide summary and suggest using get_results
if (response.data.status === 'completed' && responseSize > 15000) {
const summary = {
taskId: response.data.id,
status: response.data.status,
responseSize: responseSize,
message: "Deep research completed! Response is large. Use 'openai_deep_research_get_results' to retrieve the full results.",
outputItems: response.data.output?.length || 0,
hasWebSearchCalls: response.data.output?.some(item => item.type === 'web_search_call') || false,
hasCodeInterpreterCalls: response.data.output?.some(item => item.type === 'code_interpreter_call') || false,
hasFinalMessage: response.data.output?.some(item => item.type === 'message') || false,
outputTextPreview: response.data.output_text ?
response.data.output_text.substring(0, 500) + (response.data.output_text.length > 500 ? '...' : '')
: null
};
return {
content: [{
type: "text",
text: JSON.stringify(summary, null, 2)
}]
};
}
// For smaller responses or in-progress tasks, return full status
return {
content: [{
type: "text",
text: JSON.stringify({
taskId: response.data.id,
status: response.data.status,
...(response.data.output_text && { output_text: response.data.output_text }),
...(response.data.output && {
output: response.data.output,
outputSummary: {
totalItems: response.data.output.length,
webSearchCalls: response.data.output.filter(item => item.type === 'web_search_call').length,
codeInterpreterCalls: response.data.output.filter(item => item.type === 'code_interpreter_call').length,
messages: response.data.output.filter(item => item.type === 'message').length
}
})
}, null, 2)
}]
};
}
catch (error) {
logger.error(error);
if (axios.isAxiosError(error)) {
const errorMessage = error.response?.data?.error?.message || error.response?.data?.message || error.message;
return {
content: [{
type: "text",
text: `Failed to check deep research task status: ${errorMessage}`
}],
isError: true
};
}
return {
content: [{
type: "text",
text: `Failed to check deep research task status: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true
};
}
});
// Tool to get full deep research results
server.tool("openai_deep_research_get_results", "Get the complete results from a completed OpenAI deep research task, including final answer, citations, and tool call details.", {
taskId: z.string().describe("The task ID of the completed deep research task"),
includeToolCalls: z.boolean().default(false).describe("Include detailed tool call information (web searches, code execution) - can be very large (default: false)")
}, async ({ taskId, includeToolCalls }) => {
const requestId = `openai_deep_research_results-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`;
const logger = createRequestLogger(requestId, 'openai_deep_research_get_results');
try {
// First check cache
let taskData = deepResearchCache.get(taskId);
// If not in cache, fetch from API
if (!taskData) {
const apiKey = process.env.OPENAI_API_KEY || process.env.openaiApiKey;
if (!apiKey) {
logger.error('No OpenAI API key found');
return {
content: [{
type: "text",
text: "OPENAI_API_KEY environment variable is not set"
}],
isError: true
};
}
const url = `${OPENAI_API_BASE}/responses/${taskId}`;
logger.log(`Fetching from API: ${url}`);
const response = await axios.get(url, {
headers: {
"Authorization": `Bearer ${apiKey}`,
},
timeout: STATUS_TIMEOUT,
});
taskData = response.data;
// Cache if completed
if (taskData.status === 'completed') {
cleanupCache();
taskData._cachedAt = Date.now();
deepResearchCache.set(taskId, taskData);
}
}
logger.log(`Task status: ${taskData.status}`);
logger.complete();
if (taskData.status !== 'completed') {
return {
content: [{
type: "text",
text: JSON.stringify({
taskId: taskData.id,
status: taskData.status,
message: taskData.status === 'failed' ?
(taskData.error?.message || 'Task failed') :
'Task not yet completed. Use openai_deep_research_check_status to monitor progress.'
}, null, 2)
}]
};
}
// Extract final message with citations
const finalMessage = taskData.output?.find(item => item.type === 'message');
const citations = finalMessage?.content?.[0]?.annotations || [];
const finalText = finalMessage?.content?.[0]?.text || taskData.output_text || '';
// Build result object
const result = {
taskId: taskData.id,
status: taskData.status,
output_text: finalText,
citations: citations.length > 0 ? citations : undefined,
summary: {
totalOutputItems: taskData.output?.length || 0,
webSearchCalls: taskData.output?.filter(item => item.type === 'web_search_call').length || 0,
codeInterpreterCalls: taskData.output?.filter(item => item.type === 'code_interpreter_call').length || 0,
totalCitations: citations.length
}
};
// Include detailed tool calls if requested
if (includeToolCalls && taskData.output) {
result.toolCalls = {
webSearchCalls: taskData.output.filter(item => item.type === 'web_search_call'),
codeInterpreterCalls: taskData.output.filter(item => item.type === 'code_interpreter_call')
};
}
return {
content: [{
type: "text",
text: JSON.stringify(result, null, 2)
}]
};
}
catch (error) {
logger.error(error);
if (axios.isAxiosError(error)) {
const errorMessage = error.response?.data?.error?.message || error.response?.data?.message || error.message;
return {
content: [{
type: "text",
text: `Failed to get deep research results: ${errorMessage}`
}],
isError: true
};
}
return {
content: [{
type: "text",
text: `Failed to get deep research results: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true
};
}
});
// Tool to save deep research results to markdown
server.tool("openai_deep_research_save_to_markdown", "Save OpenAI deep research results to a markdown file with proper formatting and citations", {
taskId: z.string().describe("The task ID of the completed deep research task"),
filePath: z.string().describe("Full path where to save the markdown file (e.g., /path/to/research.md)"),
includeMetadata: z.boolean().default(true).describe("Include metadata like task ID, timestamp, etc. (default: true)"),
includeCitations: z.boolean().default(true).describe("Include citations in the markdown (default: true)"),
includeToolCalls: z.boolean().default(false).describe("Include detailed tool calls summary (default: false)")
}, async ({ taskId, filePath, includeMetadata, includeCitations, includeToolCalls }) => {
const requestId = `openai_deep_research_save_md-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`;
const logger = createRequestLogger(requestId, 'openai_deep_research_save_to_markdown');
try {
// First get the research data (from cache or API)
let taskData = deepResearchCache.get(taskId);
if (!taskData) {
const apiKey = process.env.OPENAI_API_KEY || process.env.openaiApiKey;
if (!apiKey) {
logger.error('No OpenAI API key found');
return {
content: [{
type: "text",
text: "OPENAI_API_KEY environment variable is not set"
}],
isError: true
};
}
const url = `${OPENAI_API_BASE}/responses/${taskId}`;
logger.log(`Fetching from API: ${url}`);
const response = await axios.get(url, {
headers: {
"Authorization": `Bearer ${apiKey}`,
},
timeout: STATUS_TIMEOUT,
});
taskData = response.data;
// Cache if completed
if (taskData.status === 'completed') {
cleanupCache();
taskData._cachedAt = Date.now();
deepResearchCache.set(taskId, taskData);
}
}
if (taskData.status !== 'completed') {
return {
content: [{
type: "text",
text: `Cannot save research that is not completed. Current status: ${taskData.status}`
}],
isError: true
};
}
// Extract the final research content
const finalMessage = taskData.output?.find(item => item.type === 'message');
const citations = finalMessage?.content?.[0]?.annotations || [];
const finalText = finalMessage?.content?.[0]?.text || taskData.output_text || '';
if (!finalText) {
return {
content: [{
type: "text",
text: "No research content found to save"
}],
isError: true
};
}
// Build markdown content
let markdown = '';
// Title and metadata
if (includeMetadata) {
markdown += `# OpenAI Deep Research Report\n\n`;
markdown += `**Task ID:** ${taskData.id}\n`;
markdown += `**Generated:** ${new Date().toISOString()}\n`;
markdown += `**Status:** ${taskData.status}\n`;
markdown += `**Model:** Determined by OpenAI\n\n`;
// Add tool usage summary
const webSearchCount = taskData.output?.filter(item => item.type === 'web_search_call').length || 0;
const codeInterpreterCount = taskData.output?.filter(item => item.type === 'code_interpreter_call').length || 0;
markdown += `**Research Statistics:**\n`;
markdown += `- Total output items: ${taskData.output?.length || 0}\n`;
markdown += `- Web search calls: ${webSearchCount}\n`;
markdown += `- Code interpreter calls: ${codeInterpreterCount}\n`;
markdown += `- Citations: ${citations.length}\n\n`;
markdown += `---\n\n`;
}
// Research content
markdown += `## Research Results\n\n`;
markdown += `${finalText}\n\n`;
// Citations section
if (includeCitations && citations.length > 0) {
markdown += `---\n\n`;
markdown += `## Citations\n\n`;
citations.forEach((citation, index) => {
markdown += `${index + 1}. [${citation.title || 'Source'}](${citation.url})\n`;
});
markdown += '\n';
}
// Tool calls summary
if (includeToolCalls && taskData.output) {
const webSearchCalls = taskData.output.filter(item => item.type === 'web_search_call');
const codeInterpreterCalls = taskData.output.filter(item => item.type === 'code_interpreter_call');
if (webSearchCalls.length > 0 || codeInterpreterCalls.length > 0) {
markdown += `---\n\n`;
markdown += `## Research Process\n\n`;
if (webSearchCalls.length > 0) {
markdown += `### Web Search Queries (${webSearchCalls.length})\n\n`;
webSearchCalls.slice(0, 20).forEach((call, index) => {
if (call.action && call.action.query) {
markdown += `${index + 1}. "${call.action.query}"\n`;
}
});
if (webSearchCalls.length > 20) {
markdown += `... and ${webSearchCalls.length - 20} more searches\n`;
}
markdown += '\n';
}
if (codeInterpreterCalls.length > 0) {
markdown += `### Code Analysis (${codeInterpreterCalls.length} executions)\n\n`;
markdown += `The research included ${codeInterpreterCalls.length} code interpreter executions for data analysis and verification.\n\n`;
}
}
}
// Ensure directory exists
const dir = path.dirname(filePath);
await fs.mkdir(dir, { recursive: true });
// Write the file
await fs.writeFile(filePath, markdown, 'utf-8');
logger.log(`Saved markdown to: ${filePath}`);
logger.complete();
// Get file stats
const stats = await fs.stat(filePath);
return {
content: [{
type: "text",
text: JSON.stringify({
success: true,
filePath: filePath,
fileSize: stats.size,
message: `OpenAI deep research results saved to ${filePath}`,
linesWritten: markdown.split('\n').length,
contentLength: finalText.length,
citationsIncluded: citations.length
}, null, 2)
}]
};
}
catch (error) {
logger.error(error);
if (axios.isAxiosError(error)) {
const errorMessage = error.response?.data?.error?.message || error.response?.data?.message || error.message;
return {
content: [{
type: "text",
text: `Failed to save deep research to markdown: ${errorMessage}`
}],
isError: true
};
}
return {
content: [{
type: "text",
text: `Failed to save deep research to markdown: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true
};
}
});
}