datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
554 lines • 21.2 kB
JavaScript
;
/**
* File Access Control System
* Provides secure file operations with access controls and audit logging
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.SecureFileOperations = exports.FileAccessController = void 0;
exports.getFileAccessController = getFileAccessController;
const fs_1 = require("fs");
const path_1 = require("path");
const crypto_1 = require("crypto");
const stream_1 = require("stream");
const types_1 = require("../core/types");
const logger_1 = require("../utils/logger");
const input_validator_1 = require("../utils/input-validator");
class FileAccessController {
static instance;
handles = new Map();
auditLog = [];
operationCounts = new Map();
quarantinedFiles = new Set();
defaultPolicy;
constructor() {
this.defaultPolicy = {
allowedOperations: ['read', 'metadata'],
maxFileSize: 1024 * 1024 * 1024, // 1GB
rateLimit: 60, // 60 operations per minute
requireIntegrityCheck: true,
tempFileTimeout: 300000, // 5 minutes
};
this.initializeCleanupTimer();
}
static getInstance() {
if (!FileAccessController.instance) {
FileAccessController.instance = new FileAccessController();
}
return FileAccessController.instance;
}
/**
* Create a secure file handle with validation and access control
*/
async createSecureHandle(filePath, operation, policy, context) {
try {
// Validate input using security validator
const validation = await input_validator_1.InputValidator.validateFilePath(filePath);
if (!validation.isValid) {
throw validation.errors[0] || new types_1.DataPilotError('File validation failed', 'VALIDATION_FAILED');
}
const safePath = validation.sanitizedValue;
// Check if file is quarantined
if (this.quarantinedFiles.has(safePath)) {
throw types_1.DataPilotError.security('File has been quarantined due to security concerns', 'FILE_QUARANTINED', context);
}
// Apply access policy
const effectivePolicy = {
...this.defaultPolicy,
...policy,
};
// Check if operation is allowed
if (!effectivePolicy.allowedOperations.includes(operation)) {
this.logAuditEvent({
timestamp: new Date(),
operation,
filePath: safePath,
success: false,
error: 'Operation not allowed by policy',
});
throw types_1.DataPilotError.security(`Operation '${operation}' not allowed for this file`, 'OPERATION_NOT_ALLOWED', context);
}
// Rate limiting check
if (!this.checkRateLimit(safePath, effectivePolicy.rateLimit)) {
throw types_1.DataPilotError.security('Rate limit exceeded for file operations', 'RATE_LIMIT_EXCEEDED', context);
}
// Generate file hash for integrity
let hash = '';
if (effectivePolicy.requireIntegrityCheck) {
try {
const stats = await fs_1.promises.stat(safePath);
if (stats.size <= effectivePolicy.maxFileSize) {
hash = await this.generateFileHash(safePath);
}
else {
throw types_1.DataPilotError.security(`File size (${stats.size}) exceeds policy limit (${effectivePolicy.maxFileSize})`, 'FILE_TOO_LARGE', context);
}
}
catch (error) {
if (operation === 'create' || operation === 'write') {
// For new files, hash will be generated after creation
hash = 'pending';
}
else {
throw error;
}
}
}
// Create secure handle
const handleId = this.generateHandleId();
const handle = {
id: handleId,
path: filePath,
safePath,
hash,
createdAt: Date.now(),
policy: effectivePolicy,
stats: {
reads: 0,
writes: 0,
lastAccessed: Date.now(),
},
};
this.handles.set(handleId, handle);
// Log successful handle creation
this.logAuditEvent({
timestamp: new Date(),
operation: 'create_handle',
filePath: safePath,
success: true,
metadata: { handleId, operation },
});
logger_1.logger.debug('Secure file handle created', {
handleId,
filePath: safePath,
operation,
...context,
});
return handle;
}
catch (error) {
this.logAuditEvent({
timestamp: new Date(),
operation,
filePath,
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
});
throw error;
}
}
/**
* Create a secure readable stream with access controls
*/
createSecureReadStream(handle, options) {
try {
// Verify handle is valid
this.validateHandle(handle, 'read');
// Update access statistics
handle.stats.reads++;
handle.stats.lastAccessed = Date.now();
// Create monitored read stream
const readStream = (0, fs_1.createReadStream)(handle.safePath, options);
// Add security monitoring
const securityTransform = new stream_1.Transform({
transform(chunk, encoding, callback) {
// Monitor for suspicious patterns in data
// Note: Simplified security check for compilation
callback(null, chunk);
}
});
// Log read operation
this.logAuditEvent({
timestamp: new Date(),
operation: 'read',
filePath: handle.safePath,
success: true,
metadata: { handleId: handle.id, options },
});
return readStream.pipe(securityTransform);
}
catch (error) {
this.logAuditEvent({
timestamp: new Date(),
operation: 'read',
filePath: handle.safePath,
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
});
throw error;
}
}
/**
* Create a secure writable stream with access controls
*/
createSecureWriteStream(handle, options) {
try {
// Verify handle is valid for writing
this.validateHandle(handle, 'write');
// Update access statistics
handle.stats.writes++;
handle.stats.lastAccessed = Date.now();
// Ensure output directory exists
const dir = (0, path_1.dirname)(handle.safePath);
fs_1.promises.mkdir(dir, { recursive: true }).catch(() => {
// Directory might already exist
});
// Create monitored write stream
const writeStream = (0, fs_1.createWriteStream)(handle.safePath, options);
// Monitor write operations
const originalWrite = writeStream.write.bind(writeStream);
writeStream.write = function (chunk, encoding, callback) {
// Check for malicious content
if (typeof chunk === 'string' || Buffer.isBuffer(chunk)) {
const content = chunk.toString();
if (this.detectMaliciousContent(content)) {
const error = new Error('Malicious content detected in write operation');
if (typeof callback === 'function') {
callback(error);
}
else {
this.emit('error', error);
}
return false;
}
}
return originalWrite(chunk, encoding, callback);
}.bind(this);
// Log write operation
this.logAuditEvent({
timestamp: new Date(),
operation: 'write',
filePath: handle.safePath,
success: true,
metadata: { handleId: handle.id, options },
});
// Update hash when write completes
writeStream.on('finish', async () => {
if (handle.policy.requireIntegrityCheck) {
try {
handle.hash = await this.generateFileHash(handle.safePath);
}
catch (error) {
logger_1.logger.warn('Failed to update file hash after write', {
handleId: handle.id,
error: error instanceof Error ? error.message : 'Unknown error',
});
}
}
});
return writeStream;
}
catch (error) {
this.logAuditEvent({
timestamp: new Date(),
operation: 'write',
filePath: handle.safePath,
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
});
throw error;
}
}
/**
* Verify file integrity using stored hash
*/
async verifyFileIntegrity(handle) {
try {
if (!handle.policy.requireIntegrityCheck || handle.hash === 'pending') {
return true; // No integrity check required or hash not yet generated
}
const currentHash = await this.generateFileHash(handle.safePath);
const isValid = currentHash === handle.hash;
if (!isValid) {
this.logAuditEvent({
timestamp: new Date(),
operation: 'integrity_check',
filePath: handle.safePath,
success: false,
error: 'File integrity check failed',
metadata: { expectedHash: handle.hash, actualHash: currentHash },
});
// Quarantine the file
this.quarantineFile(handle.safePath, 'Integrity check failed');
}
return isValid;
}
catch (error) {
this.logAuditEvent({
timestamp: new Date(),
operation: 'integrity_check',
filePath: handle.safePath,
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
});
return false;
}
}
/**
* Quarantine a file due to security concerns
*/
quarantineFile(filePath, reason) {
this.quarantinedFiles.add(filePath);
this.logAuditEvent({
timestamp: new Date(),
operation: 'quarantine',
filePath,
success: true,
metadata: { reason },
});
logger_1.logger.warn('File quarantined', {
filePath,
reason,
timestamp: new Date(),
});
}
/**
* Release a file from quarantine
*/
releaseFromQuarantine(filePath, authorizer) {
this.quarantinedFiles.delete(filePath);
this.logAuditEvent({
timestamp: new Date(),
operation: 'release_quarantine',
filePath,
success: true,
metadata: { authorizer },
});
logger_1.logger.info('File released from quarantine', {
filePath,
authorizer,
timestamp: new Date(),
});
}
/**
* Get audit log entries
*/
getAuditLog(filter) {
let filtered = this.auditLog;
if (filter) {
filtered = filtered.filter(entry => {
if (filter.filePath && entry.filePath !== filter.filePath)
return false;
if (filter.operation && entry.operation !== filter.operation)
return false;
if (filter.successOnly && !entry.success)
return false;
const entryTime = new Date(entry.timestamp);
if (filter.startTime && entryTime < filter.startTime)
return false;
if (filter.endTime && entryTime > filter.endTime)
return false;
return true;
});
}
return filtered;
}
/**
* Clean up expired handles and temporary files
*/
async cleanup() {
const now = Date.now();
const expiredHandles = [];
for (const [id, handle] of this.handles.entries()) {
const age = now - handle.createdAt;
if (age > handle.policy.tempFileTimeout) {
expiredHandles.push(id);
}
}
// Remove expired handles
for (const id of expiredHandles) {
this.handles.delete(id);
logger_1.logger.debug('Expired file handle removed', { handleId: id });
}
// Trim audit log if it gets too large
if (this.auditLog.length > 10000) {
this.auditLog = this.auditLog.slice(-5000); // Keep last 5000 entries
}
logger_1.logger.debug('File access controller cleanup completed', {
expiredHandles: expiredHandles.length,
totalHandles: this.handles.size,
auditLogSize: this.auditLog.length,
});
}
// Private helper methods
validateHandle(handle, operation) {
// Check if handle exists
if (!this.handles.has(handle.id)) {
throw types_1.DataPilotError.security('Invalid or expired file handle', 'INVALID_HANDLE');
}
// Check if operation is allowed
if (!handle.policy.allowedOperations.includes(operation)) {
throw types_1.DataPilotError.security(`Operation '${operation}' not allowed by handle policy`, 'OPERATION_NOT_ALLOWED');
}
// Check handle age
const age = Date.now() - handle.createdAt;
if (age > handle.policy.tempFileTimeout) {
this.handles.delete(handle.id);
throw types_1.DataPilotError.security('File handle has expired', 'HANDLE_EXPIRED');
}
}
checkRateLimit(filePath, limit) {
const key = filePath;
const now = Date.now();
const data = this.operationCounts.get(key);
if (!data || now - data.timestamp > 60000) {
// Reset or initialize counter
this.operationCounts.set(key, { count: 1, timestamp: now });
return true;
}
if (data.count >= limit) {
return false;
}
data.count++;
return true;
}
async generateFileHash(filePath) {
return new Promise((resolve, reject) => {
const hash = (0, crypto_1.createHash)('sha256');
const stream = (0, fs_1.createReadStream)(filePath);
stream.on('data', (data) => hash.update(data));
stream.on('end', () => resolve(hash.digest('hex')));
stream.on('error', reject);
});
}
generateHandleId() {
return (0, crypto_1.randomBytes)(16).toString('hex');
}
detectSuspiciousContent(chunk) {
// Check larger portion of content for security
const content = chunk.toString('utf8', 0, Math.min(chunk.length, 4096));
// Comprehensive security patterns to detect malicious content
const suspiciousPatterns = [
/\x00{10,}/, // Many null bytes
// More robust script detection patterns
/<script[\s\S]*?>/gi, // Script opening tags (including self-closing)
/<\/script>/gi, // Script closing tags
/<(iframe|object|embed|applet|form)[\s\S]*?>/gi, // Dangerous HTML elements
// Event handlers
/\bon\w+\s*=\s*["']?[\s\S]*?["']?/gi,
// Dangerous protocols
/javascript\s*:/gi,
/vbscript\s*:/gi,
// Data URLs with executable content
/data\s*:\s*text\s*\/\s*(html|javascript)/gi,
/data\s*:\s*application\s*\/\s*javascript/gi,
// Template injection patterns
/\$\{[\s\S]*?\}/g,
/\{\{[\s\S]*?\}\}/g,
// Server-side includes
/<%[\s\S]*?%>/g,
/<\?[\s\S]*?\?>/g,
];
return suspiciousPatterns.some(pattern => pattern.test(content));
}
detectMaliciousContent(content) {
// Check for various malicious patterns
const maliciousPatterns = [
/\x00/, // Null bytes
/<\?php/gi, // PHP tags
/<%[^>]*%>/g, // ASP/JSP tags
/\$\{[^}]*\}/g, // Template injection
/\beval\s*\(/gi, // eval() calls
/\bexec\s*\(/gi, // exec() calls
];
return maliciousPatterns.some(pattern => pattern.test(content));
}
logAuditEvent(entry) {
this.auditLog.push(entry);
// Also log to application logger
logger_1.logger.info('File operation audit', {
operation: entry.operation,
filePath: entry.filePath,
success: entry.success,
timestamp: entry.timestamp,
error: entry.error,
metadata: entry.metadata,
});
}
initializeCleanupTimer() {
// Run cleanup every 5 minutes
setInterval(() => {
this.cleanup().catch(error => {
logger_1.logger.error('File access controller cleanup failed', {
error: error instanceof Error ? error.message : 'Unknown error',
});
});
}, 300000);
}
/**
* Get statistics about file operations
*/
getStatistics() {
const operationCounts = {};
for (const entry of this.auditLog) {
operationCounts[entry.operation] = (operationCounts[entry.operation] || 0) + 1;
}
return {
totalHandles: this.handles.size,
quarantinedFiles: this.quarantinedFiles.size,
auditLogSize: this.auditLog.length,
operationCounts,
};
}
}
exports.FileAccessController = FileAccessController;
/**
* Factory function for easy access
*/
function getFileAccessController() {
return FileAccessController.getInstance();
}
/**
* Secure file operations wrapper
*/
class SecureFileOperations {
controller;
constructor() {
this.controller = getFileAccessController();
}
/**
* Securely read a file with access controls
*/
async readFile(filePath, options, context) {
const handle = await this.controller.createSecureHandle(filePath, 'read', options?.policy, context);
// Verify integrity before reading
const isValid = await this.controller.verifyFileIntegrity(handle);
if (!isValid) {
throw types_1.DataPilotError.security('File integrity verification failed', 'INTEGRITY_VERIFICATION_FAILED', context);
}
return new Promise((resolve, reject) => {
const chunks = [];
const stream = this.controller.createSecureReadStream(handle, {
encoding: options?.encoding,
});
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('end', () => {
const buffer = Buffer.concat(chunks);
resolve(options?.encoding ? buffer.toString(options.encoding) : buffer);
});
stream.on('error', reject);
});
}
/**
* Securely write a file with access controls
*/
async writeFile(filePath, data, options, context) {
const handle = await this.controller.createSecureHandle(filePath, 'write', {
allowedOperations: ['write', 'create'],
...options?.policy,
}, context);
return new Promise((resolve, reject) => {
const stream = this.controller.createSecureWriteStream(handle);
stream.on('finish', resolve);
stream.on('error', reject);
if (typeof data === 'string') {
stream.write(data, options?.encoding || 'utf8');
}
else {
stream.write(data);
}
stream.end();
});
}
}
exports.SecureFileOperations = SecureFileOperations;
//# sourceMappingURL=file-access-controller.js.map