@just-every/mcp-read-website-fast
Version:
Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown
239 lines (238 loc) • 8.03 kB
JavaScript
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { CallToolRequestSchema, ListToolsRequestSchema, ListResourcesRequestSchema, ReadResourceRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
process.stdin.on('error', () => { });
process.stdout.on('error', () => { });
process.stderr.on('error', () => { });
let fetchMarkdownModule;
let fsPromises;
let pathModule;
const server = new Server({
name: 'read-website-fast',
version: '0.1.0',
}, {
capabilities: {
tools: {},
resources: {},
},
});
server.onerror = error => {
console.error('[MCP Server Error]', error);
};
const READ_WEBSITE_TOOL = {
name: 'read_website_fast',
description: 'Quickly reads webpages and converts to markdown for fast, token efficient web scraping',
inputSchema: {
type: 'object',
properties: {
url: {
type: 'string',
description: 'HTTP/HTTPS URL to fetch and convert to markdown',
},
depth: {
type: 'number',
description: 'Crawl depth (0 = single page)',
default: 0,
},
respectRobots: {
type: 'boolean',
description: 'Whether to respect robots.txt',
default: true,
},
},
required: ['url'],
},
};
const RESOURCES = [
{
uri: 'read-website-fast://status',
name: 'Cache Status',
mimeType: 'application/json',
description: 'Get cache status information',
},
{
uri: 'read-website-fast://clear-cache',
name: 'Clear Cache',
mimeType: 'application/json',
description: 'Clear the cache directory',
},
];
server.setRequestHandler(ListToolsRequestSchema, async () => ({
tools: [READ_WEBSITE_TOOL],
}));
server.setRequestHandler(CallToolRequestSchema, async (request) => {
if (request.params.name !== 'read_website_fast') {
throw new Error(`Unknown tool: ${request.params.name}`);
}
try {
if (!fetchMarkdownModule) {
fetchMarkdownModule = await import('./internal/fetchMarkdown.js');
}
const args = request.params.arguments;
if (!args.url || typeof args.url !== 'string') {
throw new Error('URL parameter is required and must be a string');
}
const result = await fetchMarkdownModule.fetchMarkdown(args.url, {
depth: args.depth ?? 0,
respectRobots: args.respectRobots ?? true,
});
if (result.error && result.markdown) {
return {
content: [
{
type: 'text',
text: `${result.markdown}\n\n---\n*Note: ${result.error}*`,
},
],
};
}
if (result.error && !result.markdown) {
throw new Error(result.error);
}
return {
content: [{ type: 'text', text: result.markdown }],
};
}
catch (error) {
console.error('Tool execution error:', error);
throw new Error(`Failed to fetch content: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
});
server.setRequestHandler(ListResourcesRequestSchema, async () => ({
resources: RESOURCES,
}));
server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
const uri = request.params.uri;
if (!fsPromises) {
fsPromises = await import('fs/promises');
}
if (!pathModule) {
pathModule = await import('path');
}
if (uri === 'read-website-fast://status') {
try {
const cacheDir = '.cache';
const files = await fsPromises.readdir(cacheDir).catch(() => []);
let totalSize = 0;
for (const file of files) {
const stats = await fsPromises
.stat(pathModule.join(cacheDir, file))
.catch(() => null);
if (stats) {
totalSize += stats.size;
}
}
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify({
cacheSize: totalSize,
cacheFiles: files.length,
cacheSizeFormatted: `${(totalSize / 1024 / 1024).toFixed(2)} MB`,
}, null, 2),
},
],
};
}
catch (error) {
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify({
error: 'Failed to get cache status',
message: error instanceof Error
? error.message
: 'Unknown error',
}, null, 2),
},
],
};
}
}
if (uri === 'read-website-fast://clear-cache') {
try {
await fsPromises.rm('.cache', { recursive: true, force: true });
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify({
status: 'success',
message: 'Cache cleared successfully',
}, null, 2),
},
],
};
}
catch (error) {
return {
contents: [
{
uri,
mimeType: 'application/json',
text: JSON.stringify({
status: 'error',
message: error instanceof Error
? error.message
: 'Failed to clear cache',
}, null, 2),
},
],
};
}
}
throw new Error(`Unknown resource: ${uri}`);
});
async function runServer() {
const transport = new StdioServerTransport();
transport.onerror = error => {
console.error('[Transport Error]', error);
};
process.on('SIGINT', async () => {
console.error('Received SIGINT, shutting down gracefully...');
await server.close();
process.exit(0);
});
process.on('SIGTERM', async () => {
console.error('Received SIGTERM, shutting down gracefully...');
await server.close();
process.exit(0);
});
process.on('uncaughtException', error => {
console.error('Uncaught exception:', error);
if (error && error.message && error.message.includes('EPIPE')) {
console.error('Pipe error detected, keeping server alive');
return;
}
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => {
console.error('Unhandled rejection at:', promise, 'reason:', reason);
});
process.stdin.on('end', () => {
console.error('Stdin closed, shutting down...');
process.exit(0);
});
process.stdin.on('error', error => {
console.error('Stdin error:', error);
});
try {
await server.connect(transport);
console.error('read-website-fast MCP server running');
process.stdin.resume();
}
catch (error) {
console.error('Failed to start server:', error);
process.exit(1);
}
}
runServer().catch(error => {
console.error('Server initialization error:', error);
process.exit(1);
});