@yihuangdb/storage-object
Version:
A Node.js storage object layer library using Redis OM
524 lines • 22.7 kB
JavaScript
;
/**
* Export Import Manager
*
* Handles export and import operations for StorageObject data and schemas.
* Supports full and incremental exports, various formats, and compression.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExportImportManager = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const zlib = __importStar(require("zlib"));
const crypto_1 = require("crypto");
class ExportImportManager {
keyManager;
storageVersionManager;
redis;
constructor(keyManager, storageVersionManager, redis) {
this.keyManager = keyManager;
this.storageVersionManager = storageVersionManager;
this.redis = redis;
}
/**
* Validate and sanitize file paths to prevent directory traversal attacks
*
* @param filePath - The file path to validate
* @throws Error if the path is invalid or contains dangerous patterns
*/
validatePath(filePath) {
// Check for null or empty paths
if (!filePath || filePath.trim() === '') {
throw new Error('File path cannot be empty');
}
// Normalize the path to resolve any .. or . segments
const normalizedPath = path.normalize(filePath);
// Check for directory traversal attempts
if (normalizedPath.includes('..')) {
throw new Error('Path traversal detected - ".." is not allowed in file paths');
}
// Check for null bytes which can be used for path injection
if (filePath.includes('\0')) {
throw new Error('Invalid path - null bytes are not allowed');
}
// Check for absolute paths on Windows that might access system drives
if (process.platform === 'win32') {
// Allow paths starting with drive letters but validate them
const driveLetterRegex = /^[a-zA-Z]:\\/;
if (driveLetterRegex.test(normalizedPath)) {
// Check if it's trying to access Windows system directories
const lowerPath = normalizedPath.toLowerCase();
const dangerousPaths = ['c:\\windows', 'c:\\program files', 'c:\\programdata'];
if (dangerousPaths.some(danger => lowerPath.startsWith(danger))) {
throw new Error('Access to system directories is not allowed');
}
}
}
// Ensure the resolved path doesn't escape the intended directory
const resolvedPath = path.resolve(filePath);
const cwd = process.cwd();
// Optional: Restrict to subdirectories of current working directory
// Uncomment if you want to enforce this restriction
// if (!resolvedPath.startsWith(cwd)) {
// throw new Error('File path must be within the current working directory');
// }
}
async exportFull(storage, filePath, options = {}) {
// Validate the file path before proceeding
this.validatePath(filePath);
const exportId = `export-${Date.now()}-${Math.random().toString(36).substring(7)}`;
const exportFormat = options.exportFormat || 'json';
const shouldCompress = options.compressOutput ?? true;
const baseDir = path.dirname(filePath);
const baseName = path.basename(filePath, path.extname(filePath));
const ext = shouldCompress ? '.gz' : '';
const schemaFile = path.join(baseDir, `${baseName}-schema.json${ext}`);
const dataFile = path.join(baseDir, `${baseName}-data.${exportFormat}${ext}`);
const metaFile = path.join(baseDir, `${baseName}-meta.json`);
const [currentSchemaVersion, currentStorageVersion] = await Promise.all([
this.storageVersionManager.getCurrentSchemaVersion(),
this.storageVersionManager.getCurrentStorageVersion()
]);
let entityCount = 0;
if (options.includeSchema !== false) {
await this.exportSchema(storage, schemaFile, shouldCompress);
}
if (options.includeData !== false) {
entityCount = await this.exportData(storage, dataFile, {
...options,
exportFormat,
compressOutput: shouldCompress
});
}
const metadata = {
exportId,
exportTimestamp: Date.now(),
schemaName: storage.schemaName,
exportedSchemaVersion: currentSchemaVersion,
exportedStorageVersion: currentStorageVersion,
fromStorageVersion: options.fromStorageVersion,
toStorageVersion: options.toStorageVersion || currentStorageVersion,
exportedEntityCount: entityCount,
exportFormat,
isCompressed: shouldCompress,
isIncremental: options.incrementalExport || false,
fileChecksum: fs.existsSync(dataFile) ? await this.calculateChecksum(dataFile) : undefined
};
await fs.promises.writeFile(metaFile, JSON.stringify(metadata, null, 2));
return metadata;
}
async importFull(storage, filePath, options = {}) {
// Validate the file path before proceeding
this.validatePath(filePath);
const importId = `import-${Date.now()}-${Math.random().toString(36).substring(7)}`;
const startTime = Date.now();
const baseDir = path.dirname(filePath);
const baseName = path.basename(filePath, path.extname(filePath));
const metaFile = path.join(baseDir, `${baseName}-meta.json`);
if (!fs.existsSync(metaFile)) {
throw new Error(`Metadata file not found: ${metaFile}`);
}
const metadataStr = await fs.promises.readFile(metaFile, 'utf-8');
const metadata = JSON.parse(metadataStr);
const ext = metadata.isCompressed ? '.gz' : '';
const schemaFile = path.join(baseDir, `${baseName}-schema.json${ext}`);
const dataFile = path.join(baseDir, `${baseName}-data.${metadata.exportFormat}${ext}`);
let result = {
importId,
importedEntityCount: 0,
failedEntityCount: 0,
schemaWasUpdated: false,
startStorageVersion: await this.storageVersionManager.getCurrentStorageVersion(),
endStorageVersion: 0,
importDuration: 0,
importErrors: []
};
if (fs.existsSync(schemaFile)) {
result.schemaWasUpdated = await this.importSchema(storage, schemaFile, metadata.isCompressed, options);
}
if (fs.existsSync(dataFile)) {
const dataResult = await this.importData(storage, dataFile, {
...options,
importFormat: metadata.exportFormat,
isCompressed: metadata.isCompressed
});
result.importedEntityCount = dataResult.importedEntityCount;
result.failedEntityCount = dataResult.failedEntityCount;
result.importErrors = dataResult.importErrors;
}
else if (metadata.exportedEntityCount > 0) {
// If metadata indicates there should be data but file is missing, throw error
throw new Error(`Data file not found: ${dataFile}`);
}
result.endStorageVersion = await this.storageVersionManager.getCurrentStorageVersion();
result.importDuration = Date.now() - startTime;
return result;
}
async exportSchema(storage, filePath, compress) {
const [currentSchema, schemaVersion, schemaHistory] = await Promise.all([
storage.getSchema(),
this.storageVersionManager.getCurrentSchemaVersion(),
this.storageVersionManager.getSchemaChangesSince(0)
]);
// Get the actual field definitions from the schema object
const schemaFields = typeof currentSchema.getFields === 'function'
? currentSchema.getFields()
: currentSchema;
const schemaExport = {
currentSchemaVersion: schemaVersion,
schemaDefinition: schemaFields,
schemaChangeHistory: schemaHistory,
exportedAt: new Date().toISOString()
};
const content = JSON.stringify(schemaExport, null, 2);
if (compress) {
const gzip = zlib.createGzip();
gzip.write(content);
gzip.end();
const writeStream = fs.createWriteStream(filePath);
gzip.pipe(writeStream);
await new Promise((resolve, reject) => {
writeStream.on('finish', () => resolve());
writeStream.on('error', reject);
});
}
else {
await fs.promises.writeFile(filePath, content);
}
}
async exportData(storage, filePath, options) {
const format = options.exportFormat || 'json';
const compress = options.compressOutput ?? true;
const batchSize = options.batchSize || 1000;
let entityCount = 0;
const writeStream = fs.createWriteStream(filePath);
const outputStream = compress ? zlib.createGzip() : writeStream;
if (compress) {
outputStream.pipe(writeStream);
}
if (format === 'json') {
outputStream.write('{\n "entities": [\n');
}
if (options.incrementalExport && options.fromStorageVersion !== undefined) {
entityCount = await this.exportIncrementalData(storage, outputStream, options.fromStorageVersion, options.toStorageVersion, format, batchSize);
}
else {
entityCount = await this.exportAllEntities(storage, outputStream, format, batchSize);
}
if (format === 'json') {
outputStream.write('\n ],\n');
const exportMetadata = {
exportedAt: new Date().toISOString(),
totalEntityCount: entityCount,
currentStorageVersion: await this.storageVersionManager.getCurrentStorageVersion(),
currentSchemaVersion: await this.storageVersionManager.getCurrentSchemaVersion()
};
outputStream.write(` "exportMetadata": ${JSON.stringify(exportMetadata, null, 2)}\n}`);
}
outputStream.end();
await new Promise((resolve, reject) => {
writeStream.on('finish', () => resolve());
writeStream.on('error', reject);
outputStream.on('error', reject);
});
return entityCount;
}
async exportAllEntities(storage, stream, format, batchSize) {
let entityCount = 0;
let offset = 0;
let isFirst = true;
while (true) {
const entities = await storage.find({}, { limit: batchSize, offset });
if (entities.length === 0)
break;
for (const entity of entities) {
if (format === 'json') {
if (!isFirst)
stream.write(',\n');
stream.write(' ' + JSON.stringify(entity));
isFirst = false;
}
else if (format === 'ndjson') {
stream.write(JSON.stringify(entity) + '\n');
}
entityCount++;
}
offset += batchSize;
if (entities.length < batchSize)
break;
}
return entityCount;
}
async exportIncrementalData(storage, stream, fromStorageVersion, toStorageVersion, format, batchSize) {
let changeCount = 0;
let isFirst = true;
// Get all changes in the version range
const changes = await this.storageVersionManager.getStorageChangesBetween(fromStorageVersion, toStorageVersion);
// Group changes by entity to get latest state
const entityChanges = new Map();
for (const change of changes) {
if (change.operation === 'd') {
entityChanges.set(change.entityId, { deleted: true, change });
}
else {
entityChanges.set(change.entityId, change);
}
}
// Export each entity's final state
for (const [entityId, changeInfo] of entityChanges) {
let exportEntry;
if (changeInfo.deleted) {
exportEntry = {
changeOperation: 'delete',
entityId: entityId,
atStorageVersion: changeInfo.change.storageVersionNum
};
}
else {
const entity = await storage.findById(entityId);
if (entity) {
exportEntry = {
changeOperation: changeInfo.operation === 'c' ? 'create' : 'update',
entityData: entity,
atStorageVersion: changeInfo.storageVersionNum
};
}
}
if (exportEntry) {
if (format === 'json') {
if (!isFirst)
stream.write(',\n');
stream.write(' ' + JSON.stringify(exportEntry));
isFirst = false;
}
else if (format === 'ndjson') {
stream.write(JSON.stringify(exportEntry) + '\n');
}
changeCount++;
}
}
return changeCount;
}
async importSchema(storage, filePath, compressed, options) {
let content;
if (compressed) {
const gunzip = zlib.createGunzip();
const chunks = [];
const source = fs.createReadStream(filePath);
const decompressed = source.pipe(gunzip);
for await (const chunk of decompressed) {
chunks.push(chunk);
}
content = Buffer.concat(chunks).toString('utf-8');
}
else {
content = await fs.promises.readFile(filePath, 'utf-8');
}
const schemaExport = JSON.parse(content);
if (options.validateSchemaVersion) {
const currentSchemaVer = await this.storageVersionManager.getCurrentSchemaVersion();
if (schemaExport.currentSchemaVersion < currentSchemaVer) {
throw new Error(`Cannot import older schema version ${schemaExport.currentSchemaVersion}. ` +
`Current schema version is ${currentSchemaVer}`);
}
}
if (!options.dryRun) {
// OPTIMIZED: Use smart schema migration
// Only recreate index if indexed fields changed
await storage.updateSchemaOptimized(schemaExport.schemaDefinition);
if (schemaExport.schemaChangeHistory?.length > 0) {
for (const change of schemaExport.schemaChangeHistory) {
await this.storageVersionManager.incrementSchemaVersion(change);
}
}
}
return true;
}
async importData(storage, filePath, options) {
const format = options.importFormat || 'json';
const batchSize = options.importBatchSize || 100;
const continueOnError = options.continueOnError ?? true;
let importedCount = 0;
let failedCount = 0;
const errors = [];
const startStorageVer = await this.storageVersionManager.getCurrentStorageVersion();
let stream = fs.createReadStream(filePath);
if (options.isCompressed) {
stream = stream.pipe(zlib.createGunzip());
}
if (format === 'json') {
const chunks = [];
for await (const chunk of stream) {
chunks.push(chunk);
}
const content = Buffer.concat(chunks).toString('utf-8');
const data = JSON.parse(content);
const entities = data.entities || data;
for (let i = 0; i < entities.length; i += batchSize) {
const batch = entities.slice(i, i + batchSize);
for (const item of batch) {
// Skip null or undefined items
if (!item) {
failedCount++;
errors.push({
entityId: 'unknown',
error: 'Null or undefined entity in import data'
});
if (!continueOnError) {
throw new Error('Null or undefined entity in import data');
}
continue;
}
// Skip non-object items
if (typeof item !== 'object' || Array.isArray(item)) {
failedCount++;
errors.push({
entityId: 'unknown',
error: `Invalid entity type: expected object, got ${Array.isArray(item) ? 'array' : typeof item}`
});
if (!continueOnError) {
throw new Error(`Invalid entity type: expected object, got ${Array.isArray(item) ? 'array' : typeof item}`);
}
continue;
}
try {
if (item.changeOperation === 'delete') {
await storage.delete(item.entityId);
}
else if (item.changeOperation) {
// Incremental import
const entity = item.entityData;
const existing = await storage.findById(entity.entityId);
if (!existing) {
await storage.create(entity);
}
else {
await storage.update(entity.entityId, entity);
}
}
else {
// Full import
await this.importEntity(storage, item, options);
}
importedCount++;
}
catch (error) {
failedCount++;
errors.push({
entityId: item?.entityId || item?.entityData?.entityId || 'unknown',
error: error.message
});
if (!continueOnError)
throw error;
}
}
}
}
else if (format === 'ndjson') {
const readline = require('readline');
const rl = readline.createInterface({
input: stream,
crlfDelay: Infinity
});
let batch = [];
for await (const line of rl) {
try {
const entry = JSON.parse(line);
batch.push(entry);
if (batch.length >= batchSize) {
for (const item of batch) {
await this.importEntity(storage, item, options);
}
importedCount += batch.length;
batch = [];
}
}
catch (error) {
failedCount++;
if (!continueOnError)
throw error;
}
}
if (batch.length > 0) {
for (const item of batch) {
await this.importEntity(storage, item, options);
}
importedCount += batch.length;
}
}
const endStorageVer = await this.storageVersionManager.getCurrentStorageVersion();
return {
importedEntityCount: importedCount,
failedEntityCount: failedCount,
startStorageVersion: startStorageVer,
endStorageVersion: endStorageVer,
importErrors: errors
};
}
async importEntity(storage, item, options) {
if (options.dryRun) {
console.log(`[DRY RUN] Would import entity ${item.entityId}`);
return;
}
const mergeStrategy = options.entityMergeStrategy || 'replace';
const entity = item.entityData || item;
const existing = await storage.findById(entity.entityId);
if (existing) {
if (mergeStrategy === 'replace') {
await storage.update(entity.entityId, entity);
}
else if (mergeStrategy === 'merge') {
await storage.update(entity.entityId, { ...existing, ...entity });
}
else if (mergeStrategy === 'skip') {
return;
}
}
else {
await storage.create(entity);
}
}
async calculateChecksum(filePath) {
const hash = (0, crypto_1.createHash)('sha256');
const stream = fs.createReadStream(filePath);
for await (const chunk of stream) {
hash.update(chunk);
}
return hash.digest('hex');
}
}
exports.ExportImportManager = ExportImportManager;
//# sourceMappingURL=export-import-manager.js.map