signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
191 lines • 7.03 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.DirectoryScanner = void 0;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const path_helpers_1 = require("./path-helpers");
const cache_defaults_1 = require("../config/cache-defaults");
/**
* Cached directory scanner that reduces filesystem operations
* Maintains an index of files and only rescans when needed
*/
class DirectoryScanner {
/**
* @param cacheTTL - Cache time-to-live in milliseconds (default from config)
* @param excludedDirs - Directory names to skip during scanning (defaults to SPECIAL_DIRECTORIES)
*/
constructor(cacheTTL = cache_defaults_1.CACHE_TTL.DIRECTORY_SCAN, excludedDirs = Array.from(path_helpers_1.SPECIAL_DIRECTORIES)) {
this.cache = new Map();
this.cacheTTL = cacheTTL;
this.excludedDirs = new Set(excludedDirs);
}
/**
* Scan a directory and return all files matching the pattern
* Uses cache when available and valid
*
* @param baseDir - Root directory to scan
* @param filePattern - Optional regex pattern to match files (null = match all)
* @param forceRefresh - Force cache refresh even if valid
* @returns Array of file information
*/
async scanDirectory(baseDir, filePattern = null, forceRefresh = false) {
const cacheKey = baseDir;
const now = Date.now();
// Check cache validity
const cached = this.cache.get(cacheKey);
const isCacheValid = cached && !forceRefresh && now - cached.scannedAt < this.cacheTTL;
if (isCacheValid) {
// Filter cached results by pattern
return filePattern
? cached.files.filter(f => filePattern.test(f.name))
: cached.files;
}
// Perform scan
const files = [];
const directories = [];
await this.walkDirectory(baseDir, baseDir, files, directories);
// Update cache
this.cache.set(cacheKey, {
files,
scannedAt: now,
directories,
});
// Filter results by pattern
return filePattern ? files.filter(f => filePattern.test(f.name)) : files;
}
/**
* Find files by date pattern (e.g., "2025-11-02")
* Optimized for consolidation use case
*
* @param baseDir - Root directory to scan
* @param dateStr - Date string to match (YYYY-MM-DD format)
* @param excludeConsolidated - Exclude already consolidated files
* @returns Array of matching files
*/
async findFilesByDate(baseDir, dateStr, excludeConsolidated = true) {
const allFiles = await this.scanDirectory(baseDir);
return allFiles.filter(file => {
// Must include the date string
if (!file.name.includes(dateStr))
return false;
// Optionally exclude consolidated files
if (excludeConsolidated && file.name.includes('_consolidated')) {
return false;
}
return true;
});
}
/**
* Recursive directory walker
* Private method used by scanDirectory
*/
async walkDirectory(baseDir, currentDir, files, directories) {
try {
const items = await fs.readdir(currentDir);
for (const item of items) {
const itemPath = path.join(currentDir, item);
try {
const stat = await fs.stat(itemPath);
if (stat.isDirectory()) {
// Skip excluded directories
if (this.excludedDirs.has(item)) {
continue;
}
directories.push(itemPath);
// Recurse into subdirectory
await this.walkDirectory(baseDir, itemPath, files, directories);
}
else if (stat.isFile()) {
// Add file to results
files.push({
path: itemPath,
name: item,
size: stat.size,
modifiedTime: stat.mtimeMs,
directory: currentDir,
});
}
}
catch (statError) {
// Skip files we can't stat (permissions, etc.)
continue;
}
}
}
catch (readError) {
// Skip directories we can't read
return;
}
}
/**
* Invalidate cache for a specific directory
* Call this when files are added/removed in that directory
*/
invalidateCache(directory) {
this.cache.delete(directory);
}
/**
* Clear entire cache
* Useful for testing or manual cache refresh
*/
clearCache() {
this.cache.clear();
}
/**
* Get cache statistics
*/
getCacheStats() {
let totalFiles = 0;
let totalDirectories = 0;
this.cache.forEach(entry => {
totalFiles += entry.files.length;
totalDirectories += entry.directories.length;
});
return {
entries: this.cache.size,
totalFiles,
totalDirectories,
};
}
/**
* Get list of cached directories
*/
getCachedDirectories() {
return Array.from(this.cache.keys());
}
}
exports.DirectoryScanner = DirectoryScanner;
//# sourceMappingURL=directory-scanner.js.map