tero/dist/recovery.js

tero is a JSON document Manager, with ACID complaint and backup recovery mechanism.

import { S3Client, GetObjectCommand, ListObjectsV2Command, HeadObjectCommand } from "@aws-sdk/client-s3";
import { createWriteStream, existsSync, mkdirSync } from "fs";
import { join, dirname } from "path";
import { pipeline } from "stream/promises";
export class DataRecovery {
    s3Client;
    config;
    constructor(config) {
        this.config = config;
        this.initializeCloudStorage();
    }
    initializeCloudStorage() {
        try {
            const clientConfig = {
                region: this.config.cloudStorage.region,
                credentials: {
                    accessKeyId: this.config.cloudStorage.accessKeyId,
                    secretAccessKey: this.config.cloudStorage.secretAccessKey,
                },
            };
            // Configure for Cloudflare R2 or custom endpoints
            if (this.config.cloudStorage.endpoint) {
                clientConfig.endpoint = this.config.cloudStorage.endpoint;
                clientConfig.forcePathStyle = true; // Required for R2
            }
            this.s3Client = new S3Client(clientConfig);
        }
        catch (error) {
            throw new Error(`Failed to initialize cloud storage for recovery: ${error instanceof Error ? error.message : 'Unknown error'}`);
        }
    }
    getCloudKey(filename) {
        const prefix = this.config.cloudStorage.pathPrefix || 'tero-backups';
        const dbName = this.config.localPath.split('/').pop() || 'database';
        return `${prefix}/${dbName}/${filename}`;
    }
    async checkFileInCloud(key) {
        try {
            const cloudKey = this.getCloudKey(`${key}.json`);
            const headCommand = new HeadObjectCommand({
                Bucket: this.config.cloudStorage.bucket,
                Key: cloudKey
            });
            const response = await this.s3Client.send(headCommand);
            return {
                key,
                exists: true,
                size: response.ContentLength,
                lastModified: response.LastModified
            };
        }
        catch (error) {
            if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
                return {
                    key,
                    exists: false
                };
            }
            throw new Error(`Failed to check file in cloud: ${error.message}`);
        }
    }
    async recoverSingleFile(key) {
        try {
            const cloudKey = this.getCloudKey(`${key}.json`);
            const localFilePath = join(this.config.localPath, `${key}.json`);
            // Ensure local directory exists
            const localDir = dirname(localFilePath);
            if (!existsSync(localDir)) {
                mkdirSync(localDir, { recursive: true });
            }
            const getCommand = new GetObjectCommand({
                Bucket: this.config.cloudStorage.bucket,
                Key: cloudKey
            });
            const response = await this.s3Client.send(getCommand);
            if (!response.Body) {
                throw new Error('No data received from cloud storage');
            }
            // Stream the file to local storage
            const writeStream = createWriteStream(localFilePath);
            await pipeline(response.Body, writeStream);
            return true;
        }
        catch (error) {
            if (error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
                return false;
            }
            return false;
        }
    }
    async recoverFromArchive(archiveName) {
        const startTime = Date.now();
        const recovered = [];
        const failed = [];
        try {
            // List available archives if no specific archive is provided
            if (!archiveName) {
                const archives = await this.listAvailableArchives();
                if (archives.length === 0) {
                    throw new Error('No backup archives found in cloud storage');
                }
                // Use the most recent archive
                archiveName = archives[0];
            }
            const cloudKey = this.getCloudKey(archiveName);
            const localArchivePath = join(this.config.localPath, archiveName);
            // Download the archive
            const downloadSuccess = await this.downloadFile(cloudKey, localArchivePath);
            if (!downloadSuccess) {
                throw new Error(`Failed to download archive: ${archiveName}`);
            }
            // Extract the archive
            const extractSuccess = await this.extractArchive(localArchivePath);
            if (extractSuccess) {
                recovered.push(archiveName);
            }
            else {
                failed.push(archiveName);
            }
            const duration = Date.now() - startTime;
            return {
                success: recovered.length > 0,
                recovered,
                failed,
                totalFiles: 1,
                duration
            };
        }
        catch (error) {
            const duration = Date.now() - startTime;
            return {
                success: false,
                recovered,
                failed: [archiveName || 'unknown'],
                totalFiles: 1,
                duration
            };
        }
    }
    async recoverIndividualFiles(keys) {
        const startTime = Date.now();
        const recovered = [];
        const failed = [];
        try {
            // If no keys provided, list all available files
            if (!keys || keys.length === 0) {
                keys = await this.listAvailableFiles();
                if (keys.length === 0) {
                    throw new Error('No backup files found in cloud storage');
                }
            }
            // Recover each file
            for (const key of keys) {
                const success = await this.recoverSingleFile(key);
                if (success) {
                    recovered.push(key);
                }
                else {
                    failed.push(key);
                }
            }
            const duration = Date.now() - startTime;
            return {
                success: recovered.length > 0,
                recovered,
                failed,
                totalFiles: keys.length,
                duration
            };
        }
        catch (error) {
            const duration = Date.now() - startTime;
            return {
                success: false,
                recovered,
                failed: keys || [],
                totalFiles: keys?.length || 0,
                duration
            };
        }
    }
    async listAvailableArchives() {
        try {
            const prefix = this.getCloudKey('');
            const listCommand = new ListObjectsV2Command({
                Bucket: this.config.cloudStorage.bucket,
                Prefix: prefix,
                MaxKeys: 100
            });
            const response = await this.s3Client.send(listCommand);
            if (!response.Contents) {
                return [];
            }
            // Filter for archive files and sort by date (newest first)
            const archives = response.Contents
                .filter((obj) => obj.Key && obj.Key.endsWith('.tar.gz'))
                .sort((a, b) => {
                const dateA = a.LastModified?.getTime() || 0;
                const dateB = b.LastModified?.getTime() || 0;
                return dateB - dateA; // Newest first
            })
                .map((obj) => obj.Key.split('/').pop())
                .filter(Boolean);
            return archives;
        }
        catch (error) {
            return [];
        }
    }
    async listAvailableFiles() {
        try {
            const prefix = this.getCloudKey('');
            const listCommand = new ListObjectsV2Command({
                Bucket: this.config.cloudStorage.bucket,
                Prefix: prefix,
                MaxKeys: 1000
            });
            const response = await this.s3Client.send(listCommand);
            if (!response.Contents) {
                return [];
            }
            // Filter for JSON files and extract keys
            const files = response.Contents
                .filter((obj) => obj.Key && obj.Key.endsWith('.json'))
                .map((obj) => {
                const filename = obj.Key.split('/').pop();
                return filename.replace('.json', '');
            })
                .filter(Boolean);
            return files;
        }
        catch (error) {
            return [];
        }
    }
    async downloadFile(cloudKey, localPath) {
        try {
            const getCommand = new GetObjectCommand({
                Bucket: this.config.cloudStorage.bucket,
                Key: cloudKey
            });
            const response = await this.s3Client.send(getCommand);
            if (!response.Body) {
                return false;
            }
            // Ensure local directory exists
            const localDir = dirname(localPath);
            if (!existsSync(localDir)) {
                mkdirSync(localDir, { recursive: true });
            }
            const writeStream = createWriteStream(localPath);
            await pipeline(response.Body, writeStream);
            return true;
        }
        catch (error) {
            return false;
        }
    }
    async extractArchive(archivePath) {
        try {
            const tar = await import('tar');
            await tar.extract({
                file: archivePath,
                cwd: this.config.localPath,
                strip: 1 // Remove the top-level directory from the archive
            });
            return true;
        }
        catch (error) {
            return false;
        }
    }
    async testCloudConnection() {
        try {
            const listCommand = new ListObjectsV2Command({
                Bucket: this.config.cloudStorage.bucket,
                MaxKeys: 1
            });
            await this.s3Client.send(listCommand);
            return { success: true, message: 'Cloud storage connection successful' };
        }
        catch (error) {
            return {
                success: false,
                message: `Cloud storage connection failed: ${error instanceof Error ? error.message : 'Unknown error'}`
            };
        }
    }
    async getRecoveryInfo() {
        try {
            const cloudFiles = await this.listAvailableFiles();
            const localFiles = [];
            // Check which files exist locally
            for (const key of cloudFiles) {
                const localPath = join(this.config.localPath, `${key}.json`);
                if (existsSync(localPath)) {
                    localFiles.push(key);
                }
            }
            const missingLocally = cloudFiles.filter(key => !localFiles.includes(key));
            return {
                cloudFiles: cloudFiles.length,
                localFiles: localFiles.length,
                missingLocally,
                availableForRecovery: cloudFiles
            };
        }
        catch (error) {
            return {
                cloudFiles: 0,
                localFiles: 0,
                missingLocally: [],
                availableForRecovery: []
            };
        }
    }
}