@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
483 lines (482 loc) • 15 kB
JavaScript
/**
* Tool Cache - Caches tool results and server responses
*
* Provides intelligent caching for MCP tool calls to improve performance
* and reduce redundant operations. Supports multiple eviction strategies:
* - LRU (Least Recently Used)
* - FIFO (First In, First Out)
* - LFU (Least Frequently Used)
*/
import { createHash } from "crypto";
import { EventEmitter } from "events";
import { withTimeout } from "../../utils/async/withTimeout.js";
/**
* Tool Cache - High-performance caching for MCP tool results
*
* @example
* ```typescript
* const cache = new ToolCache({
* ttl: 60000, // 1 minute
* maxSize: 500,
* strategy: 'lru',
* });
*
* // Cache a tool result
* cache.set('getUserById:123', { id: 123, name: 'John' });
*
* // Retrieve from cache
* const user = cache.get('getUserById:123');
*
* // Invalidate by pattern
* cache.invalidate('getUserById:*');
* ```
*/
export class ToolCache extends EventEmitter {
cache = new Map();
config;
stats;
cleanupTimer;
constructor(config) {
super();
this.config = {
ttl: config.ttl,
maxSize: config.maxSize,
strategy: config.strategy,
enableAutoCleanup: config.enableAutoCleanup ?? true,
cleanupInterval: config.cleanupInterval ?? 60000,
namespace: config.namespace ?? "",
};
this.stats = {
hits: 0,
misses: 0,
evictions: 0,
size: 0,
maxSize: this.config.maxSize,
hitRate: 0,
};
if (this.config.enableAutoCleanup) {
this.startAutoCleanup();
}
}
/**
* Get a value from the cache
*
* Returns an isolated copy of the stored value (see `cloneCachedValue`), so
* a caller mutating what it gets back cannot corrupt the entry for later
* hits or for other concurrent callers of the same key.
*/
get(key) {
const fullKey = this.getFullKey(key);
const entry = this.cache.get(fullKey);
if (!entry) {
this.stats.misses++;
this.updateHitRate();
this.emit("miss", { key: fullKey });
return undefined;
}
// Check expiration
if (this.isExpired(entry)) {
this.deleteWithReason(fullKey, "expired");
this.stats.misses++;
this.updateHitRate();
this.emit("miss", { key: fullKey });
return undefined;
}
// Update access metadata
entry.accessedAt = Date.now();
entry.accessCount++;
this.stats.hits++;
this.updateHitRate();
const returnedValue = this.cloneCachedValue(entry.value);
if (this.listenerCount("hit") > 0) {
// Listeners get their own copy. `emit` is synchronous, so a listener
// that mutates `event.value` would otherwise be mutating the very object
// the caller is about to receive.
this.emit("hit", {
key: fullKey,
value: this.cloneCachedValue(entry.value),
});
}
return returnedValue;
}
/**
* Set a value in the cache
*
* Stores an isolated copy of `value` (see `cloneCachedValue`), so mutating
* the caller's original object after this call cannot reach into the
* cache entry.
*/
set(key, value, ttl) {
const fullKey = this.getFullKey(key);
const effectiveTtl = ttl ?? this.config.ttl;
const now = Date.now();
// Evict if at capacity
if (this.cache.size >= this.config.maxSize && !this.cache.has(fullKey)) {
this.evictOne();
}
const entry = {
value: this.cloneCachedValue(value),
expires: now + effectiveTtl,
createdAt: now,
accessedAt: now,
accessCount: 1,
key: fullKey,
};
this.cache.set(fullKey, entry);
this.stats.size = this.cache.size;
this.emit("set", { key: fullKey, value, ttl: effectiveTtl });
}
/**
* Check if a key exists and is not expired
*/
has(key) {
const fullKey = this.getFullKey(key);
const entry = this.cache.get(fullKey);
if (!entry) {
return false;
}
if (this.isExpired(entry)) {
this.deleteWithReason(fullKey, "expired");
return false;
}
return true;
}
/**
* Delete a specific key from the cache
*/
delete(key) {
const fullKey = this.getFullKey(key);
const deleted = this.cache.delete(fullKey);
if (deleted) {
this.stats.size = this.cache.size;
this.emit("evict", { key: fullKey, reason: "manual" });
}
return deleted;
}
/**
* Invalidate entries matching a pattern
* Supports glob-style patterns with * wildcard
*/
invalidate(pattern) {
const fullPattern = this.getFullKey(pattern);
const regex = this.patternToRegex(fullPattern);
let invalidated = 0;
for (const key of this.cache.keys()) {
if (regex.test(key)) {
this.cache.delete(key);
invalidated++;
this.emit("evict", { key, reason: "manual" });
}
}
this.stats.size = this.cache.size;
return invalidated;
}
/**
* Clear all entries from the cache
*/
clear() {
const entriesRemoved = this.cache.size;
this.cache.clear();
this.stats.size = 0;
this.emit("clear", { entriesRemoved });
}
/**
* Get or set a value (cache-aside pattern)
*/
async getOrSet(key, factory, ttl) {
const existing = this.get(key);
if (existing !== undefined) {
return existing;
}
const factoryTimeoutMs = 30_000;
const value = await withTimeout(Promise.resolve(factory()), factoryTimeoutMs, `ToolCache getOrSet factory timed out after ${factoryTimeoutMs}ms for key "${key}"`);
if (value === undefined) {
return value;
}
this.set(key, value, ttl);
return value;
}
/**
* Get cache statistics
*/
getStats() {
return { ...this.stats };
}
/**
* Reset statistics
*/
resetStats() {
this.stats.hits = 0;
this.stats.misses = 0;
this.stats.evictions = 0;
this.updateHitRate();
}
/**
* Get all keys in the cache
*/
keys() {
return Array.from(this.cache.keys());
}
/**
* Get the number of entries in the cache
*/
get size() {
return this.cache.size;
}
/**
* Generate a cache key from tool name and arguments
*/
static generateKey(toolName, args) {
const stableStringify = (value, seen = new WeakSet()) => {
if (value === null || typeof value !== "object") {
return JSON.stringify(value);
}
if (value instanceof Date) {
return `{"$date":${JSON.stringify(value.toISOString())}}`;
}
if (seen.has(value)) {
throw new TypeError("Circular structures are not supported in cache keys");
}
seen.add(value);
if (Array.isArray(value)) {
const result = "[" + value.map((v) => stableStringify(v, seen)).join(",") + "]";
seen.delete(value);
return result;
}
const sortedKeys = Object.keys(value).sort();
const entries = sortedKeys.map((k) => JSON.stringify(k) +
":" +
stableStringify(value[k], seen));
seen.delete(value);
return "{" + entries.join(",") + "}";
};
const argsHash = createHash("sha256")
.update(stableStringify(args))
.digest("hex")
.substring(0, 16);
return `${toolName}:${argsHash}`;
}
/**
* Stop the auto-cleanup timer
*/
destroy() {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = undefined;
}
this.clear();
}
// ==================== Private Methods ====================
/**
* Isolate a value crossing the cache boundary (on write into the entry,
* and on read back out of it) so no two callers — nor a caller and the
* stored entry itself — ever share object identity.
*
* Without this, `set()` stored the caller's object by reference and
* `get()` returned `entry.value` by the same reference on every hit: one
* caller mutating a result it got back (e.g. an in-place truncation or
* normalization pass) silently corrupted the entry for every later
* caller of the same key for the rest of the TTL.
*
* `structuredClone` is the primary path — it is a deep copy, has no
* caller-visible side effects, and (unlike a JSON round-trip) tolerates
* circular references, which a sufficiently deep or recursive tool
* result could contain. It throws on values it cannot clone (functions,
* some non-plain class instances); the JSON round-trip fallback covers
* that case for the plain-data shapes MCP tool results actually have
* (text/JSON content arrays), at the cost of silently dropping
* `undefined`, functions and symbol keys — acceptable for a cache that
* only ever holds serializable tool results.
*/
cloneCachedValue(value) {
if (value === null || typeof value !== "object") {
return value;
}
try {
return structuredClone(value);
}
catch {
return JSON.parse(JSON.stringify(value));
}
}
getFullKey(key) {
return this.config.namespace ? `${this.config.namespace}:${key}` : key;
}
isExpired(entry) {
return Date.now() > entry.expires;
}
/**
* Delete a cache entry by its full key with a specific eviction reason.
*/
deleteWithReason(fullKey, reason) {
const deleted = this.cache.delete(fullKey);
if (deleted) {
this.stats.evictions++;
this.stats.size = this.cache.size;
this.emit("evict", { key: fullKey, reason });
}
return deleted;
}
evictOne() {
const entryToEvict = this.selectEvictionCandidate();
if (entryToEvict) {
this.cache.delete(entryToEvict.key);
this.stats.evictions++;
this.stats.size = this.cache.size;
this.emit("evict", { key: entryToEvict.key, reason: "capacity" });
}
}
selectEvictionCandidate() {
if (this.cache.size === 0) {
return undefined;
}
switch (this.config.strategy) {
case "lru":
return this.findLRU();
case "fifo":
return this.findFIFO();
case "lfu":
return this.findLFU();
default:
return this.findLRU();
}
}
findLRU() {
let oldest;
let oldestTime = Infinity;
for (const entry of this.cache.values()) {
if (entry.accessedAt < oldestTime) {
oldestTime = entry.accessedAt;
oldest = entry;
}
}
return oldest;
}
findFIFO() {
let oldest;
let oldestTime = Infinity;
for (const entry of this.cache.values()) {
if (entry.createdAt < oldestTime) {
oldestTime = entry.createdAt;
oldest = entry;
}
}
return oldest;
}
findLFU() {
let leastFrequent;
let lowestCount = Infinity;
for (const entry of this.cache.values()) {
if (entry.accessCount < lowestCount) {
lowestCount = entry.accessCount;
leastFrequent = entry;
}
}
return leastFrequent;
}
patternToRegex(pattern) {
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
const regexPattern = escaped.replace(/\*/g, "[^:]*");
return new RegExp(`^${regexPattern}$`);
}
updateHitRate() {
const total = this.stats.hits + this.stats.misses;
this.stats.hitRate = total > 0 ? this.stats.hits / total : 0;
}
startAutoCleanup() {
this.cleanupTimer = setInterval(() => {
this.cleanupExpired();
}, this.config.cleanupInterval);
// Don't keep the process alive just for cleanup
if (this.cleanupTimer.unref) {
this.cleanupTimer.unref();
}
}
cleanupExpired() {
const now = Date.now();
for (const [key, entry] of this.cache.entries()) {
if (now > entry.expires) {
this.cache.delete(key);
this.stats.evictions++;
this.emit("evict", { key, reason: "expired" });
}
}
this.stats.size = this.cache.size;
}
}
/**
* Factory function to create a ToolCache instance
*/
export const createToolCache = (config) => new ToolCache(config);
/**
* Default cache configuration
*/
export const DEFAULT_CACHE_CONFIG = {
ttl: 5 * 60 * 1000, // 5 minutes
maxSize: 500,
strategy: "lru",
enableAutoCleanup: true,
cleanupInterval: 60000, // 1 minute
};
/**
* Tool-specific cache wrapper with automatic key generation
*/
export class ToolResultCache {
cache;
constructor(config) {
this.cache = new ToolCache({
...DEFAULT_CACHE_CONFIG,
...config,
namespace: config?.namespace ?? "tool-results",
});
}
/**
* Cache a tool result
*/
cacheResult(toolName, args, result, ttl) {
const key = ToolCache.generateKey(toolName, args);
this.cache.set(key, result, ttl);
}
/**
* Get a cached tool result
*/
getCachedResult(toolName, args) {
const key = ToolCache.generateKey(toolName, args);
return this.cache.get(key);
}
/**
* Check if a result is cached
*/
hasCachedResult(toolName, args) {
const key = ToolCache.generateKey(toolName, args);
return this.cache.has(key);
}
/**
* Invalidate all cached results for a tool
*/
invalidateTool(toolName) {
return this.cache.invalidate(`${toolName}:*`);
}
/**
* Get cache statistics
*/
getStats() {
return this.cache.getStats();
}
/**
* Clear all cached results
*/
clear() {
this.cache.clear();
}
/**
* Destroy the cache
*/
destroy() {
this.cache.destroy();
}
}
/**
* Create a tool result cache instance
*/
export const createToolResultCache = (config) => new ToolResultCache(config);