@wordbricks/fetch-mcp
Version:
Model Context Protocol (MCP) server for fetching data from the web
503 lines (495 loc) • 16.6 kB
JavaScript
// src/lib/isValidJson.ts
function isValidJson(text) {
try {
JSON.parse(text);
return true;
} catch {
return false;
}
}
// node_modules/es-toolkit/dist/string/unescape.mjs
var htmlUnescapes = {
"&": "&",
"<": "<",
">": ">",
""": '"',
"'": "'"
};
function unescape(str) {
return str.replace(/&(?:amp|lt|gt|quot|#(0+)?39);/g, (match) => htmlUnescapes[match] || "'");
}
// src/tools/convertHtmlToCollapsibleString.ts
import { z as z2 } from "zod";
// src/tools/convertJsonToCollapsibleString.ts
import { z } from "zod";
var ParamsSchema = z.object({
json: z.string(),
expand: z.boolean().default(false),
arrayMaxElements: z.number().int().min(0).default(10)
});
var convertJsonToCollapsibleString = (input) => {
const params = ParamsSchema.parse(input);
const { json, expand, arrayMaxElements } = params;
try {
const parsed = JSON.parse(json);
return stringifyWithCollapse(parsed, expand, arrayMaxElements);
} catch (error) {
return json;
}
};
function stringifyWithCollapse(value, expand, arrayMaxElements) {
if (value === null)
return "null";
if (typeof value === "string")
return JSON.stringify(value);
if (typeof value === "number" || typeof value === "boolean")
return String(value);
if (Array.isArray(value)) {
if (expand || value.length <= arrayMaxElements) {
const items = value.map((item) => stringifyWithCollapse(item, expand, arrayMaxElements));
return `[${items.join(", ")}]`;
} else {
if (arrayMaxElements === 0) {
const bottomItems = value.slice(-3);
const items = bottomItems.map((item) => stringifyWithCollapse(item, expand, arrayMaxElements));
return `[..., ${items.join(", ")}]`;
} else {
const topItems = value.slice(0, arrayMaxElements);
const bottomItems = value.slice(-3);
if (value.length <= 3) {
const items = value.map((item) => stringifyWithCollapse(item, expand, arrayMaxElements));
return `[${items.join(", ")}]`;
}
if (arrayMaxElements + 3 > value.length) {
const items = value.map((item) => stringifyWithCollapse(item, expand, arrayMaxElements));
return `[${items.join(", ")}]`;
}
const topStringified = topItems.map((item) => stringifyWithCollapse(item, expand, arrayMaxElements));
const bottomStringified = bottomItems.map((item) => stringifyWithCollapse(item, expand, arrayMaxElements));
return `[${topStringified.join(", ")}, ..., ${bottomStringified.join(", ")}]`;
}
}
}
if (typeof value === "object") {
const keys = Object.keys(value);
const pairs = keys.map((key) => `${JSON.stringify(key)}: ${stringifyWithCollapse(value[key], expand, arrayMaxElements)}`);
return `{${pairs.join(", ")}}`;
}
return String(value);
}
// src/tools/convertHtmlToCollapsibleString.ts
var ParamsSchema2 = z2.object({
html: z2.string(),
expand: z2.boolean().default(false),
arrayMaxElements: z2.number().int().min(0).default(10)
});
var convertHtmlToCollapsibleString = (input) => {
const params = ParamsSchema2.parse(input);
const { html, expand, arrayMaxElements } = params;
try {
const parsed = parseHtml(html.trim(), expand, arrayMaxElements);
return unescape(stringifyHtmlWithCollapse(parsed, expand, arrayMaxElements, false));
} catch (error) {
return html;
}
};
function parseHtml(html, expand, arrayMaxElements) {
const nodes = [];
let index = 0;
const tagsToSkip = new Set([
"head",
"style",
"noscript",
"meta",
"link",
"base",
"template",
"title",
"area",
"col",
"colgroup",
"embed",
"source",
"track",
"svg"
]);
while (index < html.length) {
const char = html[index];
if (char === "<") {
const tagEnd = html.indexOf(">", index);
if (tagEnd === -1) {
nodes.push({ type: "text", text: html.slice(index) });
break;
}
const tagContent = html.slice(index + 1, tagEnd);
if (tagContent.startsWith("!--")) {
const commentEnd = html.indexOf("-->", index);
if (commentEnd !== -1) {
index = commentEnd + 3;
continue;
}
}
if (tagContent.startsWith("/")) {
index = tagEnd + 1;
continue;
}
const parts = tagContent.split(" ");
const tagName = (parts[0] || "").toLowerCase();
if (tagName === "script") {
const scriptAttributes = {};
for (let i = 1;i < parts.length; i++) {
const part = parts[i];
if (part?.includes("=")) {
const [key, ...valueParts] = part.split("=");
if (key) {
const value = valueParts.join("=").replace(/^["']|["']$/g, "");
scriptAttributes[key.toLowerCase()] = value;
}
}
}
if (scriptAttributes.type === "application/json") {
const closingTag = `</script>`;
const closingIndex = html.indexOf(closingTag, index);
if (closingIndex !== -1) {
const scriptContent = html.slice(tagEnd + 1, closingIndex);
const processedJson = convertJsonToCollapsibleString({
json: scriptContent,
expand,
arrayMaxElements
});
const children = [{ type: "text", text: processedJson }];
nodes.push({
type: "element",
tagName: "script",
attributes: scriptAttributes,
children
});
index = closingIndex + closingTag.length;
} else {
nodes.push({
type: "element",
tagName: "script",
attributes: scriptAttributes,
children: []
});
index = tagEnd + 1;
}
} else {
const closingTag = `</script>`;
const closingIndex = html.indexOf(closingTag, index);
if (closingIndex !== -1) {
index = closingIndex + closingTag.length;
} else {
index = tagEnd + 1;
}
}
continue;
}
if (tagsToSkip.has(tagName)) {
if (tagName === "style" || tagName === "head" || tagName === "noscript" || tagName === "template" || tagName === "svg") {
const closingTag = `</${tagName}>`;
const closingIndex = html.indexOf(closingTag, index);
if (closingIndex !== -1) {
index = closingIndex + closingTag.length;
} else {
index = tagEnd + 1;
}
} else {
index = tagEnd + 1;
}
continue;
}
const isSelfClosing = ["br", "hr", "img", "input", "wbr"].includes(tagName);
const attributes = {};
for (let i = 1;i < parts.length; i++) {
const part = parts[i];
if (part?.includes("=")) {
const [key, ...valueParts] = part.split("=");
if (key) {
const value = valueParts.join("=").replace(/^["']|["']$/g, "");
attributes[key] = value;
}
}
}
if (isSelfClosing) {
nodes.push({ type: "element", tagName, attributes, children: [] });
index = tagEnd + 1;
} else {
const closingTag = `</${tagName}>`;
const closingIndex = findMatchingClosingTag(html, index, tagName);
if (closingIndex === -1) {
nodes.push({ type: "element", tagName, attributes, children: [] });
index = tagEnd + 1;
} else {
const innerHtml = html.slice(tagEnd + 1, closingIndex);
const children = parseHtml(innerHtml, expand, arrayMaxElements);
nodes.push({ type: "element", tagName, attributes, children });
index = closingIndex + closingTag.length;
}
}
} else {
const nextTag = html.indexOf("<", index);
const textEnd = nextTag === -1 ? html.length : nextTag;
const text = html.slice(index, textEnd).trim();
if (text) {
nodes.push({ type: "text", text });
}
index = textEnd;
}
}
return nodes;
}
function findMatchingClosingTag(html, startIndex, tagName) {
const openTag = `<${tagName}`;
const closeTag = `</${tagName}>`;
let depth = 1;
let index = html.indexOf(">", startIndex) + 1;
while (index < html.length && depth > 0) {
const nextOpen = html.indexOf(openTag, index);
const nextClose = html.indexOf(closeTag, index);
if (nextClose === -1) {
return -1;
}
if (nextOpen !== -1 && nextOpen < nextClose) {
depth++;
index = nextOpen + openTag.length;
} else {
depth--;
if (depth === 0) {
return nextClose;
}
index = nextClose + closeTag.length;
}
}
return -1;
}
function stringifyHtmlWithCollapse(nodes, expand, arrayMaxElements, isBodyChild = false) {
if (nodes.length === 0)
return "";
if (nodes.length === 1) {
const firstNode = nodes[0];
if (!firstNode)
return "";
return stringifyHtmlNode(firstNode, expand, arrayMaxElements, isBodyChild);
}
if (expand || isBodyChild || nodes.length <= arrayMaxElements) {
return nodes.map((node) => stringifyHtmlNode(node, expand, arrayMaxElements, false)).join("");
} else {
if (arrayMaxElements === 0) {
const bottomNodes = nodes.slice(-3);
const bottomHtml = bottomNodes.map((node) => stringifyHtmlNode(node, expand, arrayMaxElements, false)).join("");
return `<!-- ... -->${bottomHtml}`;
} else {
const topNodes = nodes.slice(0, arrayMaxElements);
const bottomNodes = nodes.slice(-3);
if (arrayMaxElements + 3 > nodes.length) {
return nodes.map((node) => stringifyHtmlNode(node, expand, arrayMaxElements, false)).join("");
}
const topHtml = topNodes.map((node) => stringifyHtmlNode(node, expand, arrayMaxElements, false)).join("");
const bottomHtml = bottomNodes.map((node) => stringifyHtmlNode(node, expand, arrayMaxElements, false)).join("");
return `${topHtml}<!-- ... -->${bottomHtml}`;
}
}
}
function stringifyHtmlNode(node, expand, arrayMaxElements, isBodyChild = false) {
if (node.type === "text") {
return node.text || "";
}
if (node.type === "comment") {
return `<!--${node.text || ""}-->`;
}
if (node.type === "element") {
const { tagName = "", attributes = {}, children = [] } = node;
const attrString = Object.entries(attributes).map(([key, value]) => `${key}="${value}"`).join(" ");
const attrPart = attrString ? ` ${attrString}` : "";
const isSelfClosing = ["br", "hr", "img", "input", "wbr"].includes(tagName);
if (isSelfClosing) {
return `<${tagName}${attrPart} />`;
}
let childrenHtml;
if (children.length === 0) {
childrenHtml = "";
} else {
const isBodyElement = tagName === "body";
if (expand || isBodyElement || children.length <= arrayMaxElements) {
childrenHtml = children.map((child) => stringifyHtmlNode(child, expand, arrayMaxElements, isBodyElement)).join("");
} else {
if (arrayMaxElements === 0) {
const bottomChildren = children.slice(-3);
const bottomHtml = bottomChildren.map((child) => stringifyHtmlNode(child, expand, arrayMaxElements, false)).join("");
childrenHtml = `<!-- ... -->${bottomHtml}`;
} else {
const topChildren = children.slice(0, arrayMaxElements);
const bottomChildren = children.slice(-3);
if (arrayMaxElements + 3 > children.length) {
childrenHtml = children.map((child) => stringifyHtmlNode(child, expand, arrayMaxElements, false)).join("");
} else {
const topHtml = topChildren.map((child) => stringifyHtmlNode(child, expand, arrayMaxElements, false)).join("");
const bottomHtml = bottomChildren.map((child) => stringifyHtmlNode(child, expand, arrayMaxElements, false)).join("");
childrenHtml = `${topHtml}<!-- ... -->${bottomHtml}`;
}
}
}
}
return `<${tagName}${attrPart}>${childrenHtml}</${tagName}>`;
}
return "";
}
// src/tools/fetch.ts
import { z as z3 } from "zod";
var DEFAULT_HEADERS = {
accept: "*/*",
"accept-language": "en-US,en;q=0.9",
"sec-ch-ua": '"Not)A;Brand";v="8", "Chromium";v="138", "Google Chrome";v="138"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "same-origin",
"sec-fetch-user": "?1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
};
var fetchTool = [
"fetch",
{
description: "Fetch data from the web",
inputSchema: {
url: z3.string().describe("The URL to fetch"),
method: z3.enum(["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"]).optional().default("GET"),
headers: z3.record(z3.string(), z3.string()).optional().describe("The headers of the request"),
body: z3.string().optional().nullable().describe("The raw text body of the request")
}
},
async ({ url, method, headers, body }) => {
const res = await fetch(url, {
method,
headers: {
...DEFAULT_HEADERS,
...headers
},
body
});
const text = await res.text();
const contentType = res.headers.get("content-type") || "";
let processedText = text;
if (contentType.includes("application/json") || isValidJson(text)) {
try {
processedText = convertJsonToCollapsibleString({
json: text,
expand: false,
arrayMaxElements: 5
});
} catch (error) {
processedText = text;
}
} else if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml")) {
try {
processedText = convertHtmlToCollapsibleString({
html: text,
expand: false,
arrayMaxElements: 10
});
} catch (error) {
processedText = text;
}
}
return {
content: [
{
type: "text",
text: JSON.stringify({
status: res.status,
headers: Object.fromEntries(res.headers.entries()),
body: processedText
})
}
]
};
}
];
// src/tools.ts
var tools = [fetchTool];
// src/lib/createMcpServer.ts
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
// package.json
var package_default = {
name: "@wordbricks/fetch-mcp",
version: "1.3.0",
description: "Model Context Protocol (MCP) server for fetching data from the web",
type: "module",
main: "./dist/lib.js",
types: "./dist/lib.d.ts",
bin: {
"fetch-mcp": "./dist/index.js"
},
exports: {
".": {
import: "./dist/lib.js",
types: "./dist/lib.d.ts"
},
"./cli": {
import: "./dist/index.js"
}
},
files: ["dist"],
publishConfig: {
access: "public"
},
keywords: [
"mcp",
"model-context-protocol",
"fetch",
"web-scraping",
"ai-tools"
],
repository: {
type: "git",
url: "https://github.com/wordbricks/fetch-mcp.git"
},
license: "MIT",
scripts: {
dev: "bun run src/index.ts",
build: "bun build src/index.ts --outdir dist --target node --format esm && bun build src/lib.ts --outdir dist --target node --format esm --external @modelcontextprotocol/sdk --external @hono/node-server --external hono --external fetch-to-node --external jsdom --external zod --external dotenv && tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
prepublishOnly: "bun run build",
test: "bun test"
},
dependencies: {
"@hono/node-server": "^1.14.4",
"@modelcontextprotocol/sdk": "^1.12.1",
dotenv: "^17.0.1",
"es-toolkit": "^1.39.3",
"fetch-to-node": "^2.1.0",
hono: "^4.7.11",
jsdom: "^26.1.0",
zod: "^3.25.50"
},
devDependencies: {
"@biomejs/biome": "^1.9.4",
"@types/bun": "^1.2.16",
"@types/jsdom": "^21.1.7",
"tsc-alias": "^1.8.16",
typescript: "^5.8.3",
"typescript-transform-paths": "^3.5.5",
vitest: "^3.2.4"
}
};
// src/lib/createMcpServer.ts
var createMcpServer = () => {
const server = new McpServer({
name: "Fetch",
description: "Fetch data from the web",
version: package_default.version
});
for (const tool of tools) {
server.registerTool(...tool);
}
return server;
};
export {
tools,
fetchTool,
createMcpServer,
convertJsonToCollapsibleString,
convertHtmlToCollapsibleString
};