@titansys/persistent-cache
Version:
A robust, enterprise-grade persistent cache with file locking, corruption detection, health monitoring, and graceful degradation.
800 lines (670 loc) • 25 kB
JavaScript
var path = require('path');
var fs = require('fs');
var mkdirp = require('mkdirp-no-bin');
// Global cleanup management to prevent multiple listeners
var globalCleanup = (function() {
var isRegistered = false;
var lockManagers = [];
function registerCleanup() {
if (isRegistered) return;
isRegistered = true;
function cleanupAll() {
lockManagers.forEach(function(manager) {
try {
manager.cleanup();
} catch (err) {
// Ignore cleanup errors
}
});
}
process.on('exit', cleanupAll);
process.on('SIGINT', function() {
cleanupAll();
process.exit(130);
});
process.on('SIGTERM', function() {
cleanupAll();
process.exit(143);
});
}
return {
addLockManager: function(manager) {
lockManagers.push(manager);
registerCleanup();
},
removeLockManager: function(manager) {
var index = lockManagers.indexOf(manager);
if (index > -1) {
lockManagers.splice(index, 1);
}
}
};
})();
// Cache locking utilities
function createLockManager(lockTimeout) {
var activeLocks = new Map();
var lockTimeout = lockTimeout || 5000; // 5 seconds default
function getLockPath(filePath) {
return filePath + '.lock';
}
function acquireLock(filePath, cb) {
var lockPath = getLockPath(filePath);
var lockId = Date.now() + '-' + Math.random().toString(36).substr(2, 9);
// Try to create lock file atomically
fs.open(lockPath, 'wx', function(err, fd) {
if (err) {
if (err.code === 'EEXIST') {
// Lock exists, check if it's stale
fs.stat(lockPath, function(statErr, stats) {
if (!statErr && Date.now() - stats.mtime.getTime() > lockTimeout) {
// Stale lock, try to remove and retry
fs.unlink(lockPath, function() {
acquireLock(filePath, cb);
});
} else {
// Active lock, retry after delay
setTimeout(function() {
acquireLock(filePath, cb);
}, 50);
}
});
} else {
cb(err);
}
return;
}
// Successfully acquired lock
fs.write(fd, lockId, function(writeErr) {
if (writeErr) {
fs.close(fd, function() {
fs.unlink(lockPath, function() {
cb(writeErr);
});
});
return;
}
fs.close(fd, function(closeErr) {
if (closeErr) {
fs.unlink(lockPath, function() {
cb(closeErr);
});
return;
}
activeLocks.set(filePath, { lockId: lockId, lockPath: lockPath });
cb(null, lockId);
});
});
});
}
function releaseLock(filePath, lockId, cb) {
var lockInfo = activeLocks.get(filePath);
if (!lockInfo || lockInfo.lockId !== lockId) {
return cb(new Error('Invalid lock'));
}
fs.unlink(lockInfo.lockPath, function(err) {
activeLocks.delete(filePath);
cb(err);
});
}
function acquireLockSync(filePath) {
var lockPath = getLockPath(filePath);
var lockId = Date.now() + '-' + Math.random().toString(36).substr(2, 9);
var startTime = Date.now();
while (Date.now() - startTime < lockTimeout) {
try {
var fd = fs.openSync(lockPath, 'wx');
fs.writeSync(fd, lockId);
fs.closeSync(fd);
activeLocks.set(filePath, { lockId: lockId, lockPath: lockPath });
return lockId;
} catch (err) {
if (err.code === 'EEXIST') {
try {
var stats = fs.statSync(lockPath);
if (Date.now() - stats.mtime.getTime() > lockTimeout) {
try {
fs.unlinkSync(lockPath);
} catch (unlinkErr) {
// Ignore unlink errors
}
}
} catch (statErr) {
// Ignore stat errors
}
// Wait a bit before retrying
var waitUntil = Date.now() + 50;
while (Date.now() < waitUntil) {
// Busy wait
}
} else {
throw err;
}
}
}
throw new Error('Failed to acquire lock within timeout');
}
function releaseLockSync(filePath, lockId) {
var lockInfo = activeLocks.get(filePath);
if (!lockInfo || lockInfo.lockId !== lockId) {
throw new Error('Invalid lock');
}
fs.unlinkSync(lockInfo.lockPath);
activeLocks.delete(filePath);
}
// Cleanup function for process exit
function cleanup() {
for (var [filePath, lockInfo] of activeLocks) {
try {
fs.unlinkSync(lockInfo.lockPath);
} catch (err) {
// Ignore cleanup errors
}
}
activeLocks.clear();
}
return {
acquireLock: acquireLock,
releaseLock: releaseLock,
acquireLockSync: acquireLockSync,
releaseLockSync: releaseLockSync,
cleanup: cleanup
};
}
// Cache validation utilities
function createValidator() {
function isValidJSON(content) {
if (typeof content !== 'string') {
return false;
}
try {
JSON.parse(content);
return true;
} catch (e) {
return false;
}
}
function validateCacheEntry(content) {
if (!isValidJSON(content)) {
return { valid: false, error: 'Invalid JSON format' };
}
var entry;
try {
entry = JSON.parse(content);
} catch (e) {
return { valid: false, error: 'JSON parse error: ' + e.message };
}
if (typeof entry !== 'object' || entry === null) {
return { valid: false, error: 'Cache entry must be an object' };
}
if (!entry.hasOwnProperty('data')) {
return { valid: false, error: 'Cache entry must have a "data" property' };
}
if (entry.hasOwnProperty('cacheUntil')) {
if (typeof entry.cacheUntil !== 'number' && entry.cacheUntil !== undefined) {
return { valid: false, error: 'cacheUntil must be a number or undefined' };
}
}
return { valid: true, entry: entry };
}
function safeParse(content, defaultValue) {
var validation = validateCacheEntry(content);
if (validation.valid) {
return { success: true, data: validation.entry };
} else {
return {
success: false,
error: validation.error,
data: defaultValue
};
}
}
return {
isValidJSON: isValidJSON,
validateCacheEntry: validateCacheEntry,
safeParse: safeParse
};
}
// Health monitoring utilities
function createHealthMonitor(cacheDir, validator, options, persist) {
var monitorOptions = options || {};
var healthCheckInterval = monitorOptions.healthCheckInterval || 300000; // 5 minutes default
var autoRepair = monitorOptions.autoRepair || false;
var healthCheckTimer = null;
var isMonitoring = false;
function scanCacheFiles(cb) {
if (!persist) {
return cb(null, { healthy: 0, corrupted: 0, repaired: 0, files: [] });
}
fs.readdir(cacheDir, function(err, files) {
if (err) {
return cb(err);
}
var cacheFiles = files.filter(function(file) {
return file.endsWith('.json') && !file.endsWith('.lock');
});
var results = {
healthy: 0,
corrupted: 0,
repaired: 0,
files: []
};
var completed = 0;
var total = cacheFiles.length;
if (total === 0) {
return cb(null, results);
}
cacheFiles.forEach(function(file) {
var filePath = path.join(cacheDir, file);
var fileName = file.slice(0, -5); // Remove .json extension
fs.readFile(filePath, 'utf8', function(readErr, content) {
completed++;
if (readErr) {
results.files.push({
name: fileName,
status: 'error',
error: readErr.message
});
} else {
var validation = validator.validateCacheEntry(content);
if (validation.valid) {
results.healthy++;
results.files.push({
name: fileName,
status: 'healthy'
});
} else {
results.corrupted++;
var fileResult = {
name: fileName,
status: 'corrupted',
error: validation.error
};
if (autoRepair) {
fs.unlink(filePath, function(unlinkErr) {
if (!unlinkErr) {
results.repaired++;
fileResult.status = 'repaired';
}
});
}
results.files.push(fileResult);
}
}
if (completed === total) {
cb(null, results);
}
});
});
});
}
function scanCacheFilesSync() {
if (!persist) {
return { healthy: 0, corrupted: 0, repaired: 0, files: [] };
}
try {
var files = fs.readdirSync(cacheDir);
var cacheFiles = files.filter(function(file) {
return file.endsWith('.json') && !file.endsWith('.lock');
});
var results = {
healthy: 0,
corrupted: 0,
repaired: 0,
files: []
};
cacheFiles.forEach(function(file) {
var filePath = path.join(cacheDir, file);
var fileName = file.slice(0, -5); // Remove .json extension
try {
var content = fs.readFileSync(filePath, 'utf8');
var validation = validator.validateCacheEntry(content);
if (validation.valid) {
results.healthy++;
results.files.push({
name: fileName,
status: 'healthy'
});
} else {
results.corrupted++;
var fileResult = {
name: fileName,
status: 'corrupted',
error: validation.error
};
if (autoRepair) {
try {
fs.unlinkSync(filePath);
results.repaired++;
fileResult.status = 'repaired';
} catch (unlinkErr) {
// Failed to repair
}
}
results.files.push(fileResult);
}
} catch (readErr) {
results.files.push({
name: fileName,
status: 'error',
error: readErr.message
});
}
});
return results;
} catch (err) {
throw err;
}
}
function startMonitoring(onHealthCheck) {
if (isMonitoring || !persist) {
return;
}
isMonitoring = true;
healthCheckTimer = setInterval(function() {
scanCacheFiles(function(err, results) {
if (onHealthCheck) {
onHealthCheck(err, results);
}
});
}, healthCheckInterval);
}
function stopMonitoring() {
if (healthCheckTimer) {
clearInterval(healthCheckTimer);
healthCheckTimer = null;
}
isMonitoring = false;
}
return {
scanCacheFiles: scanCacheFiles,
scanCacheFilesSync: scanCacheFilesSync,
startMonitoring: startMonitoring,
stopMonitoring: stopMonitoring,
isMonitoring: function() { return isMonitoring; }
};
}
function exists(dir) {
try {
fs.accessSync(dir);
} catch(err) {
return false;
}
return true;
}
function safeCb(cb) {
if(typeof cb === 'function')
return cb;
return function(){};
}
function cache(options) {
var options = options || {};
var base = path.normalize(
(options.base || (require.main ? path.dirname(require.main.filename) : undefined) || process.cwd()) + '/cache'
);
var cacheDir = path.normalize(base + '/' + (options.name || 'cache'));
var cacheInfinitely = !(typeof options.duration === "number");
var cacheDuration = options.duration;
var ram = typeof options.memory == 'boolean' ? options.memory : true;
var persist = typeof options.persist == 'boolean' ? options.persist : true;
// Initialize lock manager for file operations
var lockManager = createLockManager(options.lockTimeout);
// Initialize validator for cache entry validation
var validator = createValidator();
// Initialize health monitor for cache file monitoring
var healthMonitor = createHealthMonitor(cacheDir, validator, options, persist);
// Register with global cleanup manager
if (persist) {
globalCleanup.addLockManager(lockManager);
}
if(ram)
var memoryCache = {};
if(persist && !exists(cacheDir))
mkdirp.sync(cacheDir);
function buildFilePath(name) {
return path.normalize(cacheDir + '/' + name + '.json');
}
function buildCacheEntry(data) {
return {
cacheUntil: !cacheInfinitely ? new Date().getTime() + cacheDuration : undefined,
data: data
};
}
function put(name, data, cb = () => {}) {
var entry = buildCacheEntry(data);
var filePath = buildFilePath(name);
if(persist) {
lockManager.acquireLock(filePath, function(lockErr, lockId) {
if (lockErr) {
return safeCb(cb)(lockErr);
}
fs.writeFile(filePath, JSON.stringify(entry), function(writeErr) {
lockManager.releaseLock(filePath, lockId, function(unlockErr) {
if (writeErr) return safeCb(cb)(writeErr);
if (unlockErr) return safeCb(cb)(unlockErr);
safeCb(cb)(null);
});
});
});
}
if(ram) {
// Create a separate entry for memory cache to avoid mutation
var memEntry = buildCacheEntry(data);
memEntry.data = JSON.stringify(memEntry.data);
memoryCache[name] = memEntry;
if(!persist)
return safeCb(cb)(null);
}
}
function putSync(name, data) {
var entry = buildCacheEntry(data);
var filePath = buildFilePath(name);
if(persist) {
var lockId = lockManager.acquireLockSync(filePath);
try {
fs.writeFileSync(filePath, JSON.stringify(entry));
} finally {
lockManager.releaseLockSync(filePath, lockId);
}
}
if(ram) {
memoryCache[name] = entry;
memoryCache[name].data = JSON.stringify(memoryCache[name].data);
}
}
function get(name, cb) {
if(ram && !!memoryCache[name]) {
var entry = memoryCache[name];
if(!!entry.cacheUntil && new Date().getTime() > entry.cacheUntil) {
return safeCb(cb)(null, undefined);
}
try{
entry = JSON.parse(entry.data);
}
catch(e){
// Log parsing error for memory cache but continue gracefully
if (typeof console !== 'undefined' && console.warn) {
console.warn('Memory cache parse error for', name + ':', e.message);
}
return safeCb(cb)(null, undefined);
}
return safeCb(cb)(null, entry);
}
fs.readFile(buildFilePath(name), 'utf8' ,onFileRead);
function onFileRead(err, content) {
if(err != null) {
return safeCb(cb)(null, undefined);
}
var parseResult = validator.safeParse(content);
if (!parseResult.success) {
// Log validation error for debugging but return undefined to gracefully degrade
if (typeof console !== 'undefined' && console.warn) {
console.warn('Cache validation error for', name + ':', parseResult.error);
}
return safeCb(cb)(null, undefined);
}
var entry = parseResult.data;
if(!!entry.cacheUntil && new Date().getTime() > entry.cacheUntil) {
return safeCb(cb)(null, undefined);
}
return safeCb(cb)(null, entry.data);
}
}
function getSync(name) {
if(ram && !!memoryCache[name]) {
var entry = memoryCache[name];
if(entry.cacheUntil && new Date().getTime() > entry.cacheUntil) {
return undefined;
}
try {
return JSON.parse(entry.data);
} catch(e) {
// Log parsing error for memory cache but continue gracefully
if (typeof console !== 'undefined' && console.warn) {
console.warn('Memory cache parse error for', name + ':', e.message);
}
return undefined;
}
}
try {
var content = fs.readFileSync(buildFilePath(name), 'utf8');
var parseResult = validator.safeParse(content);
if (!parseResult.success) {
// Log validation error for debugging but return undefined to gracefully degrade
if (typeof console !== 'undefined' && console.warn) {
console.warn('Cache validation error for', name + ':', parseResult.error);
}
return undefined;
}
var data = parseResult.data;
if(data.cacheUntil && new Date().getTime() > data.cacheUntil)
return undefined;
return data.data;
} catch(e) {
return undefined;
}
}
function deleteEntry(name, cb = () => {}) {
var filePath = buildFilePath(name);
if(ram) {
delete memoryCache[name];
if(!persist)
return safeCb(cb)(null);
}
if(persist) {
lockManager.acquireLock(filePath, function(lockErr, lockId) {
if (lockErr) {
return safeCb(cb)(lockErr);
}
fs.unlink(filePath, function(unlinkErr) {
lockManager.releaseLock(filePath, lockId, function(unlockRelErr) {
// Prioritize the unlink error, but still report lock release errors
safeCb(cb)(unlinkErr || unlockRelErr);
});
});
});
}
}
function deleteEntrySync(name) {
var filePath = buildFilePath(name);
if(ram) {
delete memoryCache[name];
if(!persist)
return;
}
if(persist) {
var lockId = lockManager.acquireLockSync(filePath);
try {
fs.unlinkSync(filePath);
} finally {
lockManager.releaseLockSync(filePath, lockId);
}
}
}
function unlink(cb) {
if(persist) {
// Remove from global cleanup manager
globalCleanup.removeLockManager(lockManager);
// Use built-in fs.rm with recursive option for Node.js 14.14.0+
if (fs.rm) {
return fs.rm(cacheDir, { recursive: true, force: true }, safeCb(cb));
}
// Fallback for older Node.js versions
else if (fs.rmdir && fs.rmdir.length > 2) {
// Node.js 12.10.0+ has recursive option
return fs.rmdir(cacheDir, { recursive: true }, safeCb(cb));
}
else {
// Manual recursive deletion for very old Node.js versions
return unlinkRecursive(cacheDir, safeCb(cb));
}
}
safeCb(cb)(null);
}
function unlinkRecursive(dirPath, cb) {
fs.readdir(dirPath, function(err, files) {
if (err) return cb(err);
var pending = files.length;
if (pending === 0) {
return fs.rmdir(dirPath, cb);
}
files.forEach(function(file) {
var filePath = path.join(dirPath, file);
fs.stat(filePath, function(statErr, stats) {
if (statErr) return cb(statErr);
if (stats.isDirectory()) {
unlinkRecursive(filePath, function(rmErr) {
if (rmErr) return cb(rmErr);
pending--;
if (pending === 0) {
fs.rmdir(dirPath, cb);
}
});
} else {
fs.unlink(filePath, function(unlinkErr) {
if (unlinkErr) return cb(unlinkErr);
pending--;
if (pending === 0) {
fs.rmdir(dirPath, cb);
}
});
}
});
});
});
}
function transformFileNameToKey(fileName) {
return fileName.slice(0, -5);
}
function keys(cb) {
cb = safeCb(cb);
if(ram && !persist)
return cb(null, Object.keys(memoryCache));
fs.readdir(cacheDir, onDirRead);
function onDirRead(err, files) {
return !!err ? cb(err) : cb(err, files.map(transformFileNameToKey));
}
}
function keysSync() {
if(ram && !persist)
return Object.keys(memoryCache);
return fs.readdirSync(cacheDir).map(transformFileNameToKey);
}
return {
put: put,
get: get,
delete: deleteEntry,
putSync: putSync,
getSync: getSync,
deleteSync: deleteEntrySync,
keys: keys,
keysSync: keysSync,
unlink: unlink,
// Health monitoring API
healthCheck: healthMonitor.scanCacheFiles,
healthCheckSync: healthMonitor.scanCacheFilesSync,
startHealthMonitoring: healthMonitor.startMonitoring,
stopHealthMonitoring: healthMonitor.stopMonitoring,
isHealthMonitoring: healthMonitor.isMonitoring
};
}
module.exports = cache;