mcp-think-tank
Version:
Structured thinking and knowledge management tool for Model Context Protocol
232 lines (231 loc) • 8.98 kB
JavaScript
import { createHash } from 'crypto';
import * as fs from 'fs/promises';
import { LRUCache } from 'lru-cache';
import * as path from 'path';
import * as http from 'http';
import * as https from 'https';
// Environment variables
const CACHE_CONTENT = process.env.CACHE_CONTENT !== 'false'; // Default to true
const CONTENT_CACHE_SIZE = parseInt(process.env.CONTENT_CACHE_SIZE || '100', 10);
const CONTENT_CACHE_TTL = parseInt(process.env.CONTENT_CACHE_TTL || '300000', 10); // 5 minutes in ms
/**
* ContentCache provides caching for file and URL content based on SHA-1 hashing
* This implements Story 3-G: Execution cache for file/URL reads
*/
export class ContentCache {
constructor() {
// Stats for monitoring
this.hits = 0;
this.misses = 0;
// Initialize LRU caches
this.contentCache = new LRUCache({
max: CONTENT_CACHE_SIZE,
ttl: CONTENT_CACHE_TTL,
});
this.pathHashCache = new LRUCache({
max: CONTENT_CACHE_SIZE,
ttl: CONTENT_CACHE_TTL,
});
}
/**
* Calculate SHA-1 hash of content
* @param content Content to hash
* @returns SHA-1 hash as hex string
*/
calculateHash(content) {
const hash = createHash('sha1');
hash.update(content);
return hash.digest('hex');
}
/**
* Read a file with caching
* @param filePath Path to the file to read
* @returns File content as a Buffer
*/
async readFile(filePath) {
if (!CACHE_CONTENT) {
// If caching is disabled, just read the file
return await fs.readFile(filePath);
}
try {
const normalizedPath = path.normalize(filePath);
// Check if we have the file's content hash in cache
const cachedContentHash = this.pathHashCache.get(normalizedPath);
if (cachedContentHash) {
// Check if content is in cache
const cachedContent = this.contentCache.get(cachedContentHash);
if (cachedContent) {
// Check if file has changed by comparing modification time
try {
const stats = await fs.stat(normalizedPath);
const modifiedTime = stats.mtime.getTime();
// If the cached content's timestamp is greater than or equal to the file's mtime,
// the cached content is still valid (file hasn't been modified since we cached it)
if (cachedContent.timestamp && modifiedTime <= cachedContent.timestamp) {
this.hits++;
console.log(`[ContentCache] Cache hit for file: ${filePath}`);
return cachedContent.data;
}
else {
// File has been modified, we need to re-read it
console.log(`[ContentCache] File modified, re-reading: ${filePath}`);
}
}
catch (statError) {
// If stat fails, we'll just fall back to reading the file
console.log(`[ContentCache] Could not stat file: ${filePath}, falling back to cache`);
this.hits++;
return cachedContent.data;
}
}
}
// Cache miss - read the file
this.misses++;
console.log(`[ContentCache] Cache miss for file: ${filePath}`);
try {
const fileContent = await fs.readFile(normalizedPath);
const contentHash = this.calculateHash(fileContent);
// Store content hash by file path
this.pathHashCache.set(normalizedPath, contentHash);
// Store content by hash
this.contentCache.set(contentHash, {
data: fileContent,
timestamp: Date.now(),
source: normalizedPath
});
return fileContent;
}
catch (readError) {
console.error(`[ContentCache] Error reading file: ${filePath}`, readError);
throw readError;
}
}
catch (error) {
console.error(`[ContentCache] Error reading file: ${filePath}`, error);
throw error;
}
}
/**
* Fetch URL with caching
* @param url URL to fetch
* @returns Response data
*/
async fetchUrl(url) {
if (!CACHE_CONTENT) {
// If caching is disabled, just fetch the URL
return await this.httpGet(url);
}
try {
// Check if we have the URL's content hash in cache
const cachedContentHash = this.pathHashCache.get(url);
if (cachedContentHash) {
// Check if content is in cache
const cachedContent = this.contentCache.get(cachedContentHash);
if (cachedContent) {
const now = Date.now();
const age = now - (cachedContent.timestamp || 0);
// Use cached content if it's fresh enough
if (age < CONTENT_CACHE_TTL) {
this.hits++;
console.log(`[ContentCache] Cache hit for URL: ${url}`);
return cachedContent.data;
}
}
}
// Cache miss - fetch the URL
this.misses++;
console.log(`[ContentCache] Cache miss for URL: ${url}`);
const responseData = await this.httpGet(url);
const contentHash = this.calculateHash(responseData);
// Store content hash by URL
this.pathHashCache.set(url, contentHash);
// Store content by hash
this.contentCache.set(contentHash, {
data: responseData,
timestamp: Date.now(),
source: url
});
return responseData;
}
catch (error) {
console.error(`[ContentCache] Error fetching URL: ${url}`, error);
throw error;
}
}
/**
* Helper method to get HTTP content using native Node.js http/https modules
* @param url URL to fetch
* @returns Promise resolving to Buffer with content
*/
httpGet(url) {
return new Promise((resolve, reject) => {
const isHttps = url.startsWith('https://');
const client = isHttps ? https : http;
client.get(url, (response) => {
if (response.statusCode && (response.statusCode < 200 || response.statusCode >= 300)) {
return reject(new Error(`Status Code: ${response.statusCode}`));
}
const chunks = [];
response.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
response.on('end', () => resolve(Buffer.concat(chunks)));
}).on('error', reject);
});
}
/**
* Get cached parsed JSON for a file or URL
* Parses JSON only once and caches the parsed result
* @param source File path or URL
* @param isUrl Whether the source is a URL
* @returns Parsed JSON object
*/
async getJson(source, isUrl = false) {
const content = isUrl
? await this.fetchUrl(source)
: await this.readFile(source);
// Generate a unique key for the parsed JSON
const contentHash = this.calculateHash(content);
const jsonCacheKey = `json:${contentHash}`;
// Check if we have parsed this JSON before
let parsed = this.contentCache.get(jsonCacheKey);
if (parsed) {
console.log(`[ContentCache] Parsed JSON cache hit for: ${source}`);
return parsed.data;
}
// Parse JSON and cache it
try {
const parsedData = JSON.parse(content.toString('utf8'));
// Store parsed JSON
this.contentCache.set(jsonCacheKey, {
data: parsedData,
timestamp: Date.now(),
source
});
return parsedData;
}
catch (error) {
console.error(`[ContentCache] Error parsing JSON from: ${source}`, error);
throw error;
}
}
/**
* Clear all caches
*/
clear() {
this.contentCache.clear();
this.pathHashCache.clear();
this.hits = 0;
this.misses = 0;
}
/**
* Get cache statistics
*/
getStats() {
return {
hits: this.hits,
misses: this.misses,
cacheSize: this.contentCache.size
};
}
}
// Export a singleton instance
export const contentCache = new ContentCache();