@anyshift/mcp-tools-common
Version:
Reusable JQ tool and file writing utilities for MCP servers
580 lines (569 loc) • 24.2 kB
JavaScript
import fs from 'fs/promises';
import path2 from 'path';
import crypto from 'crypto';
import { spawn } from 'child_process';
import { existsSync, realpathSync } from 'fs';
import { z } from 'zod';
// src/fileWriter/writer.ts
var generateCompactTimestamp = () => {
const now = /* @__PURE__ */ new Date();
const epoch = Math.floor(now.getTime() / 1e3);
const ms = now.getMilliseconds().toString().padStart(3, "0");
return `${epoch}${ms}`;
};
var hashArgs = (args) => {
const argsString = Object.entries(args).filter(
([key, value]) => (
// Filter out sensitive or irrelevant keys
!["api_key", "app_key", "token", "api_token"].includes(key.toLowerCase()) && value !== void 0
)
).map(([key, value]) => `${key}=${value}`).join("_");
if (!argsString) return "noargs";
return crypto.createHash("md5").update(argsString).digest("hex").substring(0, 6);
};
var generateCompactFilename = (toolName, args, toolAbbreviations) => {
const timestamp = generateCompactTimestamp();
const toolAbbrev = toolAbbreviations?.[toolName] || toolName.substring(0, 6);
const argsHash = hashArgs(args);
return `${timestamp}_${toolAbbrev}_${argsHash}.json`;
};
// src/fileWriter/schema.ts
function analyzeJsonSchema(obj, path4 = "root") {
if (obj === null) return { type: "null" };
if (obj === void 0) return { type: "undefined" };
const type = Array.isArray(obj) ? "array" : typeof obj;
if (type === "object") {
const properties = {};
const objRecord = obj;
const keys = Object.keys(objRecord);
const numericKeys = keys.filter((k) => /^\d+$/.test(k));
const hasNumericKeys = keys.length > 0 && numericKeys.length >= keys.length * 0.8;
for (const key in objRecord) {
if (Object.prototype.hasOwnProperty.call(objRecord, key)) {
properties[key] = analyzeJsonSchema(objRecord[key], `${path4}.${key}`);
}
}
const schema = { type: "object", properties };
if (hasNumericKeys) {
schema._keysAreNumeric = true;
schema._accessPattern = 'Use .["0"] not .[0]';
}
return schema;
} else if (type === "array") {
const arr = obj;
if (arr.length === 0) {
return { type: "array", items: { type: "unknown" }, length: 0 };
}
const itemTypes = /* @__PURE__ */ new Set();
let hasNulls = false;
const sampled = arr.slice(0, Math.min(10, arr.length));
for (const item of sampled) {
if (item === null) {
hasNulls = true;
itemTypes.add("null");
} else {
itemTypes.add(Array.isArray(item) ? "array" : typeof item);
}
}
const schema = {
type: "array",
items: itemTypes.size === 1 && !hasNulls ? analyzeJsonSchema(arr[0], `${path4}[0]`) : { types: Array.from(itemTypes) },
length: arr.length
};
if (hasNulls) {
schema._hasNulls = true;
}
return schema;
} else {
return { type };
}
}
function extractNullableFields(schema, basePath = "") {
const alwaysNull = [];
const nullable = [];
function traverse(s, path4) {
if (!s || typeof s !== "object") return;
const schemaObj = s;
if (schemaObj.type === "null") {
alwaysNull.push(path4);
return;
}
if (schemaObj.items && typeof schemaObj.items === "object") {
const items = schemaObj.items;
if (items.types && Array.isArray(items.types) && items.types.includes("null")) {
nullable.push(path4);
}
}
if (schemaObj.type === "object" && schemaObj.properties) {
const props = schemaObj.properties;
for (const [key, value] of Object.entries(props)) {
const newPath = path4 ? `${path4}.${key}` : key;
traverse(value, newPath);
}
}
if (schemaObj.type === "array" && schemaObj.items && typeof schemaObj.items === "object") {
const items = schemaObj.items;
if (items.type === "object" && items.properties) {
const props = items.properties;
for (const [key, value] of Object.entries(props)) {
const newPath = path4 ? `${path4}[].${key}` : `[].${key}`;
traverse(value, newPath);
}
}
}
}
traverse(schema, basePath);
return { alwaysNull, nullable };
}
// src/fileWriter/writer.ts
var DEFAULT_MIN_CHARS = 1e3;
var isErrorResponse = (response) => {
if (!response || typeof response !== "object") return false;
const obj = response;
if (obj.error || obj.isError) {
return true;
}
const rawText = extractRawText(obj);
if (rawText) {
try {
let jsonText = rawText;
const jsonMatch = rawText.match(/:\s*(\{.*\}|\[.*\])$/s);
if (jsonMatch) {
jsonText = jsonMatch[1];
}
const parsed = JSON.parse(jsonText);
return parsed.status === "error";
} catch {
return rawText.includes('"status":"error"') || rawText.includes('"status": "error"');
}
}
return obj.status === "error";
};
var extractErrorMessage = (response) => {
if (!response || typeof response !== "object") {
return "Unknown error occurred";
}
const obj = response;
if (obj.error && typeof obj.error === "string") {
return obj.error;
}
const rawText = extractRawText(obj);
if (rawText) {
try {
let jsonText = rawText;
const jsonMatch = rawText.match(/:\s*(\{.*\}|\[.*\])$/s);
if (jsonMatch) {
jsonText = jsonMatch[1];
}
const parsed = JSON.parse(jsonText);
if (parsed.error && typeof parsed.error === "string") {
return parsed.error;
}
if (parsed.message && typeof parsed.message === "string") {
return parsed.message;
}
if (parsed.status === "error") {
return "API request failed with error status";
}
} catch {
const errorMatch = rawText.match(/"error":\s*"([^"]+)"/);
if (errorMatch && errorMatch[1]) {
return errorMatch[1];
}
const messageMatch = rawText.match(/"message":\s*"([^"]+)"/);
if (messageMatch && messageMatch[1]) {
return messageMatch[1];
}
}
}
return "API error occurred";
};
var extractRawText = (response) => {
if (response._rawText) {
return response._rawText;
}
if (response.content && Array.isArray(response.content)) {
const textContent = response.content.filter((item) => item.type === "text").map((item) => item.text).join("\n");
return textContent || null;
}
return null;
};
async function handleToolResponse(config, toolName, args, responseData) {
if (toolName === "execute_jq_query") {
return responseData;
}
if (isErrorResponse(responseData)) {
const errorMessage = extractErrorMessage(responseData);
return {
content: [
{
type: "text",
text: `Error: ${errorMessage}`
}
],
isError: true
};
}
if (!config.enabled || !config.outputPath) {
return responseData;
}
const rawText = extractRawText(
responseData
);
let contentLength = 0;
if (rawText) {
contentLength = rawText.length;
} else if (responseData && typeof responseData === "object" && "content" in responseData && Array.isArray(responseData.content)) {
const textContent = responseData.content.filter((item) => item.type === "text").map((item) => item.text).join("\n");
contentLength = textContent.length;
} else {
contentLength = JSON.stringify(responseData).length;
}
const minChars = config.minCharsForWrite ?? DEFAULT_MIN_CHARS;
if (contentLength < minChars) {
return responseData;
}
try {
const filename = generateCompactFilename(
toolName,
args,
config.toolAbbreviations
);
const filepath = path2.join(config.outputPath, filename);
await fs.mkdir(config.outputPath, { recursive: true });
let contentToWrite;
let parsedForSchema;
if (rawText) {
try {
let jsonText = rawText;
const jsonMatch = rawText.match(/:\s*(\{.*\}|\[.*\])$/s);
if (jsonMatch) {
jsonText = jsonMatch[1];
}
const parsed = JSON.parse(jsonText);
parsedForSchema = parsed;
const { pagination, has_more, next_page, previous_page, page, page_size, total_pages, ...cleanData } = parsed;
contentToWrite = JSON.stringify(cleanData, null, 2);
} catch {
contentToWrite = rawText;
}
} else {
if (responseData && typeof responseData === "object" && "content" in responseData && Array.isArray(responseData.content)) {
const textContent = responseData.content.filter((item) => item.type === "text").map((item) => item.text).join("\n");
contentToWrite = textContent;
} else {
contentToWrite = JSON.stringify(responseData, null, 2);
}
}
await fs.writeFile(filepath, contentToWrite);
let schemaInfo = "";
let quickReference = "";
if (parsedForSchema) {
const { pagination, has_more, next_page, previous_page, page, page_size, total_pages, ...cleanData } = parsedForSchema;
const schema = analyzeJsonSchema(cleanData);
const nullFields = extractNullableFields(schema);
const schemaObj = schema;
quickReference += `
\u{1F50D} UNDERSTAND THIS SCHEMA BEFORE WRITING JQ QUERIES:
`;
if (schemaObj._keysAreNumeric) {
quickReference += ` \u2022 Structure: Object with numeric keys ("0", "1", ...) - use .["0"]
`;
} else if (schemaObj.type === "array") {
quickReference += ` \u2022 Structure: Array with ${schemaObj.length} items
`;
} else if (schemaObj.type === "object" && schemaObj.properties) {
const props = schemaObj.properties;
const keys = Object.keys(props).slice(0, 5).join(", ");
quickReference += ` \u2022 Structure: Object with keys: ${keys}
`;
}
if (nullFields.alwaysNull.length > 0) {
const fieldList = nullFields.alwaysNull.slice(0, 5).join(", ");
const more = nullFields.alwaysNull.length > 5 ? ` (+${nullFields.alwaysNull.length - 5} more)` : "";
quickReference += ` \u2022 Always null: ${fieldList}${more}
`;
}
if (nullFields.nullable.length > 0) {
const fieldList = nullFields.nullable.slice(0, 5).join(", ");
const more = nullFields.nullable.length > 5 ? ` (+${nullFields.nullable.length - 5} more)` : "";
quickReference += ` \u2022 Sometimes null: ${fieldList}${more}
`;
}
if (schemaObj._keysAreNumeric) {
quickReference += ` \u2022 Explore: keys, .["0"] | keys, .["0"]
`;
} else if (schemaObj.type === "array" && schemaObj.length > 0) {
quickReference += ` \u2022 Explore: length, .[0] | keys, .[0]
`;
} else {
quickReference += ` \u2022 Explore: keys, type
`;
}
schemaInfo = `
Full JSON Schema:
${JSON.stringify(schema, null, 2)}`;
}
const lineCount = contentToWrite.split("\n").length;
return {
content: [
{
type: "text",
text: `\u{1F4C4} File: ${filepath}
Size: ${contentToWrite.length} characters | Lines: ${lineCount}${quickReference}${schemaInfo}`
}
]
};
} catch (error) {
console.error(`[handleToolResponse] Error writing file:`, error);
return responseData;
}
}
// src/fileWriter/index.ts
function createFileWriter(config) {
return {
/**
* Handle tool response - writes to file if conditions are met
* @param toolName - Name of the tool that generated the response
* @param args - Arguments passed to the tool
* @param responseData - The response data to potentially write to file
* @returns Either the original response or a file reference response
*/
handleResponse: async (toolName, args, responseData) => {
return handleToolResponse(config, toolName, args, responseData);
}
};
}
var validatePathWithinAllowedDirs = (filePath, allowedPaths) => {
if (!path2.isAbsolute(filePath)) {
throw new Error(
`Absolute path required. Received: ${filePath}. Paths must start with "/" to prevent ambiguity.`
);
}
const absolutePath = path2.resolve(filePath);
if (!existsSync(absolutePath)) {
throw new Error(`File does not exist: ${filePath}`);
}
let realPath;
try {
realPath = realpathSync(absolutePath);
} catch (error) {
throw new Error(`Cannot resolve path: ${filePath}`);
}
for (const allowedPath of allowedPaths) {
const allowedPathReal = realpathSync(path2.resolve(allowedPath));
if (realPath.startsWith(allowedPathReal + path2.sep) || realPath === allowedPathReal) {
console.error(
`[validatePathWithinAllowedDirs] Path allowed (within ${allowedPathReal}): ${realPath}`
);
return realPath;
}
}
const allowedPathsList = allowedPaths.join(", ");
throw new Error(
`Access denied: File path must be within allowed directories (${allowedPathsList}). Attempted path: ${realPath}`
);
};
// src/jq/handler.ts
var DEFAULT_TIMEOUT_MS = 3e4;
async function executeJqQuery(config, jqQuery, filePath) {
if (!jqQuery || !filePath) {
throw new Error("jq_query and file_path are required");
}
const dangerousPatterns = [
/\$ENV/i,
// $ENV variable access
/env\./i,
// env.VARIABLE access
/@env/i,
// @env function
/\.env\[/i,
// .env["VARIABLE"] access
/getenv/i,
// getenv function
/\$__loc__/i,
// location info that might leak paths
/input_filename/i
// input filename access
];
const isDangerous = dangerousPatterns.some(
(pattern) => pattern.test(jqQuery)
);
if (isDangerous) {
throw new Error(
"The jq query contains patterns that could access environment variables or system information. Please use a different query."
);
}
if (!path2.isAbsolute(filePath)) {
throw new Error(
`File path must be an absolute path starting with "/": ${filePath}`
);
}
if (!existsSync(filePath)) {
throw new Error(`File not found: ${filePath}`);
}
if (!filePath.toLowerCase().endsWith(".json")) {
throw new Error(
`Only JSON files (.json) are supported for jq processing: ${filePath}`
);
}
validatePathWithinAllowedDirs(filePath, config.allowedPaths);
const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
return new Promise((resolve, reject) => {
const jqProcess = spawn("jq", [jqQuery, filePath], {
stdio: ["pipe", "pipe", "pipe"],
timeout: timeoutMs
});
let stdout = "";
let stderr = "";
jqProcess.stdout.on("data", (data) => {
stdout += data.toString();
});
jqProcess.stderr.on("data", (data) => {
stderr += data.toString();
});
jqProcess.on("close", (code) => {
if (code === 0) {
const responseText = stdout.trim();
resolve({
content: [
{
type: "text",
text: responseText
}
]
});
} else {
reject(
new Error(
`jq command failed with exit code ${code}: ${stderr.trim()}`
)
);
}
});
jqProcess.on("error", (error) => {
reject(new Error(`Failed to execute jq command: ${error.message}`));
});
setTimeout(() => {
if (!jqProcess.killed) {
jqProcess.kill("SIGTERM");
reject(new Error(`jq command timed out after ${timeoutMs}ms`));
}
}, timeoutMs);
});
}
var ExecuteJqQuerySchema = z.object({
jq_query: z.string().describe(
"The jq query to execute on the JSON file. Query will be sanitized to prevent environment variable access."
),
file_path: z.string().describe(
'Absolute path starting with "/" pointing to the JSON file to process. Must be a valid, existing file with .json extension. The file will be validated for existence and readability before processing.'
),
description: z.string().optional().describe(
"Optional description; a short explanation of the purpose of the query"
)
});
var JQ_TOOL_DEFINITION = {
name: "execute_jq_query",
description: 'Execute a jq query on a JSON file with comprehensive debugging and retry strategies.This tool processes JSON files using jq operations for data extraction, transformation, and filtering. \n\n\u26A0\uFE0F **CRITICAL SYNTAX RULE**: The jq_query parameter is a STRING. Use PLAIN quotes like .["0"] - DO NOT ESCAPE them as .["0"] or .[\\"0\\"]. The MCP framework handles escaping automatically.\n\n\u{1F4CB} **SCHEMA-FIRST WORKFLOW**: READ the schema from the file write response. For nullable fields use: select(.field != null) or .field // "default"\n\nCRITICAL: When queries fail or return null/empty results, DO NOT abandon the jq approach immediately. Instead, follow the debugging workflow below.\n\n## JQ SYNTAX REFERENCE (Fix Common Errors):\n**QUOTE SYNTAX (MOST COMMON ERROR):**\n- The jq_query is passed as a STRING to the jq binary\n- Use PLAIN quotes: .["key"] NOT .["key"] or .[\\"key\\"]\n- DO NOT escape quotes - the parameter is already a string\n- \u2705 CORRECT: .["0"], .["field"], .field\n- \u274C WRONG: .["0"], .[\\"field\\"], .\\"field\\"\n\n**ARRAY SLICING (Not Shell Commands):**\n- First N items: .[:5] NOT head -5 or head(5)\n- Last N items: .[-5:] NOT tail -5\n- Skip first N: .[5:] NOT tail +6\n- With limit: limit(10; .) NOT head(10)\n- \u2705 CORRECT: .[:10], limit(20; .), .[-5:]\n- \u274C WRONG: head -10, head(20), tail -5\n\n**COMMON FUNCTION MISTAKES:**\n- \u274C head, tail, wc, cut, grep, awk, sed \u2192 These are SHELL commands, not jq!\n- \u2705 .[:N] \u2192 Take first N (array slice)\n- \u2705 limit(N; .) \u2192 Limit to N items\n- \u2705 length \u2192 Count items\n- \u2705 sort_by(.field) \u2192 Sort by field\n- \u2705 group_by(.field) \u2192 Group by field\n\n**STRING OPERATIONS:**\n- Regex: test("pattern") NOT test(\'pattern\')\n- Contains: contains("text") or test("text")\n- Split: split(",")\n- Join: join(",")\n\n**OBJECT CONSTRUCTION:** Arithmetic needs extra parens: {count: ((.items | length) + 1)} or use variables\n**TYPE SAFETY:** Check schema types. Error "Cannot index X with Y" = wrong type, use: type == "object"\n\n## DEBUGGING WORKFLOW (Use when queries fail or return unexpected results):\n1. **Structure Inspection**: Start with `keys`, `type`, `length` to understand the data structure\n2. **Single Record Access**: Test `.["0"]`, `.["1"]` to access specific items (for object with numeric string keys)\n3. **Field Exploration**: Check `.["0"].Values`, `.["0"].Keys` to understand nested structures\n4. **Build Incrementally**: Start with simple queries, add complexity step by step\n5. **Test Without Filters**: Remove `select()` clauses to see if data exists\n\n## CYPHER RESULT STRUCTURES (Common patterns from Neo4j/graph queries):\n- **Objects with numeric string keys**: {"0": {...}, "1": {...}, "2": {...}} - Use `.["0"]`, NOT `.[0]`\n- **Values/Keys arrays**: Each result has `.Values` and `.Keys` arrays with corresponding data\n- **Null handling**: Many Values arrays contain null - always check for null before processing\n\n## OBJECT vs ARRAY ACCESS PATTERNS:\n\u2705 **For objects with numeric keys**: `.["0"].Values`, `to_entries[] | .value`, `.[keys[0]]`\n\u274C **Wrong**: `.[0].Values` (treats object as array - will fail)\n\u2705 **Iteration over objects**: `to_entries[] | .value | ...` or `.[] | ...`\n\u2705 **Safe null checking**: `select(.Values[0] != null)` or `select(.Values // [] | length > 0)`\n\n## RETRY STRATEGIES (Try multiple approaches):\n1. **Different iteration methods**: `.[]`, `to_entries[] | .value`, `.[keys[]]`\n2. **Incremental filtering**: Start with no filters, add conditions one by one\n3. **Alternative null handling**: Use `// empty`, `select(. != null)`, or `try ... catch`\n4. **Simplified queries**: Break complex queries into smaller, testable parts\n\n## COMPREHENSIVE EXAMPLES:\n**Debugging sequence for Cypher results**:\n- `keys` \u2192 ["0", "1", "2", ...] (shows object structure)\n- `.["0"]` \u2192 {"Values": [...], "Keys": [...]} (shows single result format)\n- `.["0"].Values` \u2192 [value1, value2, null, ...] (shows actual data)\n- `to_entries[] | .value | select(.Values[0] != null)` \u2192 final query\n\n**Common transformations**:\n- Extract non-null records: `to_entries[] | .value | select(.Values[0] != null)`\n- Build objects from Values/Keys: `.["0"] | {(.Keys[0]): .Values[0], (.Keys[1]): .Values[1]}`\n- Count results: `to_entries | length` or `keys | length`\n- Filter by position: `to_entries[] | select(.key | tonumber < 5) | .value`\n\n**IMPORTANT**: If a query returns null/empty, try simpler versions first to verify data exists before assuming the approach is wrong.',
inputSchema: {
type: "object",
properties: {
jq_query: {
type: "string",
description: "The jq query to execute on the JSON file. Query will be sanitized to prevent environment variable access."
},
file_path: {
type: "string",
description: 'Absolute path starting with "/" pointing to the JSON file to process. Must be a valid, existing file with .json extension. The file will be validated for existence and readability before processing.'
},
description: {
type: "string",
description: "Optional description; a short explanation of the purpose of the query"
}
},
required: ["jq_query", "file_path"]
}
};
// src/jq/index.ts
function createJqTool(config) {
return {
/**
* Tool definition for MCP server registration
*/
toolDefinition: JQ_TOOL_DEFINITION,
/**
* Handler for JQ query execution requests
* @param request - MCP request containing jq_query and file_path
* @returns Promise with the query result
*/
handler: async (request) => {
const { jq_query, file_path } = ExecuteJqQuerySchema.parse(
request.params.arguments
);
return executeJqQuery(config, jq_query, file_path);
}
};
}
// src/truncation/truncate.ts
var DEFAULT_CHARS_PER_TOKEN = 4;
function estimateTokens(text, charsPerToken = DEFAULT_CHARS_PER_TOKEN) {
return Math.ceil(text.length / charsPerToken);
}
function wouldBeTruncated(content, maxTokens, charsPerToken = DEFAULT_CHARS_PER_TOKEN) {
return estimateTokens(content, charsPerToken) > maxTokens;
}
function truncateResponseIfNeeded(config, content) {
const charsPerToken = config.charsPerToken || DEFAULT_CHARS_PER_TOKEN;
const estimatedTokens = estimateTokens(content, charsPerToken);
const contentLength = content.length;
const maxChars = config.maxTokens * charsPerToken;
if (config.enableLogging) {
process.stderr.write(
JSON.stringify({
event: "truncation_check",
content_length: contentLength,
estimated_tokens: estimatedTokens,
max_token_limit: config.maxTokens,
max_chars: maxChars,
will_truncate: estimatedTokens > config.maxTokens
}) + "\n"
);
}
if (estimatedTokens <= config.maxTokens) {
if (config.enableLogging) {
process.stderr.write(
JSON.stringify({
event: "no_truncation_needed",
estimated_tokens: estimatedTokens,
limit: config.maxTokens
}) + "\n"
);
}
return content;
}
const truncated = content.substring(0, maxChars);
const truncatedTokens = estimateTokens(truncated, charsPerToken);
if (config.enableLogging) {
process.stderr.write(
JSON.stringify({
event: "content_truncated",
original_length: contentLength,
original_tokens: estimatedTokens,
truncated_length: truncated.length,
truncated_tokens: truncatedTokens,
limit: config.maxTokens
}) + "\n"
);
}
const messagePrefix = config.messagePrefix || "RESPONSE TRUNCATED";
return `=== ${messagePrefix} ===
Estimated tokens: ${estimatedTokens} (limit: ${config.maxTokens})
Response truncated to prevent context overflow.
Please refine your query to be more specific and fetch less data.
=== END TRUNCATION NOTICE ===
${truncated}`;
}
export { ExecuteJqQuerySchema, JQ_TOOL_DEFINITION, analyzeJsonSchema, createFileWriter, createJqTool, estimateTokens, executeJqQuery, extractNullableFields, generateCompactFilename, handleToolResponse, truncateResponseIfNeeded, validatePathWithinAllowedDirs, wouldBeTruncated };
//# sourceMappingURL=index.js.map
//# sourceMappingURL=index.js.map