react-native-sst-storage-db/lib/SSTStorage.js

A powerful file-based MongoDB-like database for React Native. Single JSON file storage with collections, advanced querying, indexing, and atomic operations. No AsyncStorage dependency.

/**
 * SSTStorage - File-Based JSON Database
 * A MongoDB-like local storage library for React Native
 *
 * Features:
 * - Single JSON file database
 * - Collection-based document storage
 * - MongoDB-like query operations
 * - ACID-like transactions with atomic file writes
 * - Indexing for performance
 * - Schema migrations
 * - Cross-platform compatibility
 */

const RNFS = require("react-native-fs");
const Collection = require("./Collection");
const QueryEngine = require("./QueryEngine");
const DatabaseUtils = require("./DatabaseUtils");
const path = require("path");

class SSTStorage {
  constructor(options = {}) {
    // Database state
    this.database = {
      metadata: {
        version: "2.0.0",
        created: null,
        lastModified: null,
        collections: [],
        schema: {},
      },
      collections: {},
      indexes: {},
      keyValueStore: {},
    };

    this.initialized = false;
    this.collections = new Map(); // Collection instance cache

    // Configuration options
    this.options = {
      databaseName: options.databaseName || "SSTStorage",
      databasePath: options.databasePath || null, // Will be auto-generated
      autoSave: options.autoSave !== false, // Default true
      enableIndexing: options.enableIndexing !== false, // Default true
      enableLogging: options.enableLogging || false,
      maxCollections: options.maxCollections || 100,
      maxDocuments: options.maxDocuments || 50000,
      backupCount: options.backupCount || 3,
      encryptData: options.encryptData || false,
      compression: options.compression || false,
      ...options,
    };

    // Set database file path - use the most persistent directory available
    // Priority: ExternalDirectoryPath (most persistent) > DocumentDirectoryPath > MainBundlePath
    this.databasePath = this.options.databasePath || this._getPersistentPath();

    this.log("📦 SSTStorage created with file-based architecture", {
      databasePath: this.databasePath,
      options: this.options,
      selectedDirectory: this._getSelectedDirectoryInfo(),
    });
  }

  /**
   * Get the most persistent directory path for storing database
   * Priority: ExternalDirectoryPath > DocumentDirectoryPath > MainBundlePath
   */
  _getPersistentPath() {
    // Try ExternalDirectoryPath first (most persistent on Android)
    if (RNFS.ExternalDirectoryPath) {
      return `${RNFS.ExternalDirectoryPath}/${this.options.databaseName}.json`;
    }
    
    // Fallback to DocumentDirectoryPath
    if (RNFS.DocumentDirectoryPath) {
      return `${RNFS.DocumentDirectoryPath}/${this.options.databaseName}.json`;
    }
    
    // Last resort - MainBundlePath (not ideal for persistence)
    if (RNFS.MainBundlePath) {
      return `${RNFS.MainBundlePath}/${this.options.databaseName}.json`;
    }
    
    // If none available, throw error
    throw new Error('No suitable directory found for persistent storage');
  }

  /**
   * Get information about the selected directory for debugging
   */
  _getSelectedDirectoryInfo() {
    const info = {
      available: {
        ExternalDirectoryPath: !!RNFS.ExternalDirectoryPath,
        DocumentDirectoryPath: !!RNFS.DocumentDirectoryPath,
        MainBundlePath: !!RNFS.MainBundlePath,
      },
      selected: 'unknown'
    };

    if (RNFS.ExternalDirectoryPath && this.databasePath.includes(RNFS.ExternalDirectoryPath)) {
      info.selected = 'ExternalDirectoryPath';
    } else if (RNFS.DocumentDirectoryPath && this.databasePath.includes(RNFS.DocumentDirectoryPath)) {
      info.selected = 'DocumentDirectoryPath';
    } else if (RNFS.MainBundlePath && this.databasePath.includes(RNFS.MainBundlePath)) {
      info.selected = 'MainBundlePath';
    }

    return info;
  }

  /**
   * Verify that the selected directory is writable
   */
  async _verifyDirectoryWritable() {
    try {
      const testFile = `${this.databasePath}.test`;
      const testContent = 'test';
      
      // Try to write a test file
      await RNFS.writeFile(testFile, testContent, 'utf8');
      
      // Try to read it back
      const readContent = await RNFS.readFile(testFile, 'utf8');
      
      if (readContent !== testContent) {
        throw new Error('Directory verification failed: content mismatch');
      }
      
      // Clean up test file
      await RNFS.unlink(testFile);
      
      this.log("✅ Directory verification successful", {
        directory: path.dirname(this.databasePath),
        writable: true
      });
      
    } catch (error) {
      this.log("❌ Directory verification failed", {
        directory: path.dirname(this.databasePath),
        error: error.message
      });
      
      // Try to fallback to a different directory
      await this._fallbackToAlternativeDirectory();
    }
  }

  /**
   * Fallback to an alternative directory if the current one is not writable
   */
  async _fallbackToAlternativeDirectory() {
    this.log("🔄 Attempting to fallback to alternative directory...");
    
    const originalPath = this.databasePath;
    const originalDir = this._getSelectedDirectoryInfo().selected;
    
    // Try different directories in order of preference
    const alternatives = [
      { name: 'ExternalDirectoryPath', path: RNFS.ExternalDirectoryPath },
      { name: 'DocumentDirectoryPath', path: RNFS.DocumentDirectoryPath },
      { name: 'MainBundlePath', path: RNFS.MainBundlePath }
    ];
    
    for (const alt of alternatives) {
      if (alt.path && alt.name !== originalDir) {
        try {
          const newPath = `${alt.path}/${this.options.databaseName}.json`;
          const testFile = `${newPath}.test`;
          
          await RNFS.writeFile(testFile, 'test', 'utf8');
          await RNFS.unlink(testFile);
          
          // If we get here, this directory works
          this.databasePath = newPath;
          this.log("✅ Fallback successful", {
            from: originalDir,
            to: alt.name,
            newPath: newPath
          });
          return;
          
        } catch (error) {
          this.log(`⚠️ Fallback to ${alt.name} failed:`, error.message);
        }
      }
    }
    
    throw new Error('No writable directory found for database storage');
  }

  /**
   * Logging utility
   */
  log(message, data = null) {
    if (this.options.enableLogging) {
      if (data) {
        console.log(message, data);
      } else {
        console.log(message);
      }
    }
  }

  /**
   * Clean up any leftover temporary files
   */
  async _cleanupTempFiles() {
    try {
      const dbDir = path.dirname(this.databasePath);
      const files = await RNFS.readDir(dbDir);

      // Find and remove temporary files
      const tempFiles = files.filter(
        (file) =>
          file.name.includes(".tmp.") &&
          file.name.startsWith(this.options.databaseName)
      );

      for (const tempFile of tempFiles) {
        try {
          // Add retry logic for iOS file deletion
          let retryCount = 0;
          const maxRetries = 3;
          
          while (retryCount < maxRetries) {
            try {
              await RNFS.unlink(tempFile.path);
              this.log(`🧹 Cleaned up temp file: ${tempFile.name}`);
              break; // Success, exit retry loop
            } catch (deleteError) {
              retryCount++;
              if (retryCount >= maxRetries) {
                this.log(`⚠️ Failed to cleanup temp file ${tempFile.name} after ${maxRetries} retries:`, deleteError);
                break;
              }
              
              // Wait before retrying
              await new Promise((resolve) => setTimeout(resolve, 100 * retryCount));
            }
          }
        } catch (error) {
          this.log(`⚠️ Failed to cleanup temp file ${tempFile.name}:`, error);
        }
      }
    } catch (error) {
      this.log("⚠️ Failed to cleanup temp files:", error);
    }
  }

  /**
   * Initialize the database
   * Loads existing JSON file or creates new database
   */
  async initialize() {
    try {
      this.log("🚀 Initializing file-based SSTStorage database...");

      // Verify the selected directory is writable
      await this._verifyDirectoryWritable();

      // Ensure database directory exists
      const dbDir = path.dirname(this.databasePath);
      await RNFS.mkdir(dbDir);

      // Clean up any leftover temporary files
      await this._cleanupTempFiles();

      // Add a small delay for iOS file system stability
      await new Promise((resolve) => setTimeout(resolve, 50));

      // Check if database file exists
      const exists = await RNFS.exists(this.databasePath);

      if (exists) {
        // Load existing database
        this.log("📂 Loading existing database file...");
        
        try {
          const fileContent = await RNFS.readFile(this.databasePath, "utf8");
          this.database = JSON.parse(fileContent);

          // Validate database structure
          await this._validateAndMigrateDatabase();

          this.log("✅ Database loaded successfully", {
            collections: Object.keys(this.database.collections).length,
            keyValuePairs: Object.keys(this.database.keyValueStore).length,
            totalDocuments: this._getTotalDocumentCount(),
          });
        } catch (parseError) {
          this.log(
            "⚠️ Database file corrupted, creating backup and starting fresh..."
          );

          // Create backup of corrupted file
          const backupPath = this.databasePath + ".corrupted." + Date.now();
          try {
            await RNFS.copyFile(this.databasePath, backupPath);
          } catch (backupError) {
            this.log("⚠️ Failed to create backup:", backupError);
          }

          // Create new database
          await this._createNewDatabase();
        }
      } else {
        // Create new database
        this.log("🆕 Creating new database file...");
        await this._createNewDatabase();
      }

      this.initialized = true;
      this.log("✅ SSTStorage initialized successfully!");

      return true;
    } catch (error) {
      this.log("❌ SSTStorage initialization failed:", error);
      
      // If initialization fails, try to recover by creating a fresh database
      try {
        this.log("🔄 Attempting recovery by creating fresh database...");
        await this._createNewDatabase();
        this.initialized = true;
        this.log("✅ SSTStorage recovery successful!");
        return true;
      } catch (recoveryError) {
        this.log("❌ SSTStorage recovery failed:", recoveryError);
        throw error; // Throw original error
      }
    }
  }

  /**
   * Check if storage is initialized
   */
  isInitialized() {
    return this.initialized;
  }

  /**
   * Create a new database with default structure
   */
  async _createNewDatabase() {
    const now = new Date().toISOString();

    this.database = {
      metadata: {
        version: "2.0.0",
        created: now,
        lastModified: now,
        collections: [],
        schema: {},
      },
      collections: {},
      indexes: {},
      keyValueStore: {},
    };

    await this._saveDatabase();
    this.log("✅ New database created successfully");
  }

  /**
   * Validate and migrate database structure
   */
  async _validateAndMigrateDatabase() {
    let needsSave = false;

    // Ensure required structure exists
    if (!this.database.metadata) {
      this.database.metadata = {
        version: "2.0.0",
        created: new Date().toISOString(),
        lastModified: new Date().toISOString(),
        collections: [],
        schema: {},
      };
      needsSave = true;
    }

    if (!this.database.collections) {
      this.database.collections = {};
      needsSave = true;
    }

    if (!this.database.indexes) {
      this.database.indexes = {};
      needsSave = true;
    }

    if (!this.database.keyValueStore) {
      this.database.keyValueStore = {};
      needsSave = true;
    }

    // Update metadata
    this.database.metadata.collections = Object.keys(this.database.collections);
    this.database.metadata.lastModified = new Date().toISOString();

    if (needsSave) {
      await this._saveDatabase();
      this.log("✅ Database structure validated and migrated");
    }
  }

  /**
   * Save database to file with atomic write
   */
  async _saveDatabase() {
    if (!this.options.autoSave) {
      return;
    }

    let tempPath = null;

    try {
      // Update metadata
      this.database.metadata.lastModified = new Date().toISOString();
      this.database.metadata.collections = Object.keys(
        this.database.collections
      );

      // Create unique temporary file path to avoid conflicts
      const timestamp = Date.now();
      const randomSuffix = Math.random().toString(36).substring(2, 8);
      tempPath = this.databasePath + `.tmp.${timestamp}.${randomSuffix}`;

      // Check if temp file already exists and remove it
      const tempExists = await RNFS.exists(tempPath);
      if (tempExists) {
        await RNFS.unlink(tempPath);
      }

      const jsonData = JSON.stringify(this.database, null, 2);

      // Write to temporary file first (atomic write)
      await RNFS.writeFile(tempPath, jsonData, "utf8");

      // Atomic rename (move temp file to actual file)
      // Enhanced retry logic for iOS file operations with better error handling
      let retryCount = 0;
      const maxRetries = 5;

      while (retryCount < maxRetries) {
        try {
          // For iOS, first check if target file exists and remove it if necessary
          const targetExists = await RNFS.exists(this.databasePath);
          if (targetExists) {
            await RNFS.unlink(this.databasePath);
          }
          
          await RNFS.moveFile(tempPath, this.databasePath);
          break; // Success, exit retry loop
        } catch (moveError) {
          retryCount++;
          
          // Check if it's a "file already exists" error on iOS
          if (moveError.message && moveError.message.includes('already exists')) {
            this.log(`⚠️ File already exists error, cleaning up and retrying (${retryCount}/${maxRetries})`);
            
            // Try to clean up the target file
            try {
              const targetExists = await RNFS.exists(this.databasePath);
              if (targetExists) {
                await RNFS.unlink(this.databasePath);
              }
            } catch (cleanupError) {
              this.log("⚠️ Failed to cleanup target file:", cleanupError);
            }
          }
          
          if (retryCount >= maxRetries) {
            throw moveError; // Re-throw if max retries reached
          }

          // Wait a bit before retrying (exponential backoff with jitter)
          const delay = 100 * retryCount + Math.random() * 50;
          await new Promise((resolve) => setTimeout(resolve, delay));
          
          this.log(
            `⚠️ Move file failed, retrying (${retryCount}/${maxRetries}):`,
            moveError.message
          );
        }
      }

      this.log("💾 Database saved successfully");
    } catch (error) {
      this.log("❌ Failed to save database:", error);

      // Clean up temporary file if it exists
      if (tempPath) {
        try {
          const tempExists = await RNFS.exists(tempPath);
          if (tempExists) {
            await RNFS.unlink(tempPath);
          }
        } catch (cleanupError) {
          this.log("⚠️ Failed to cleanup temp file:", cleanupError);
        }
      }

      throw error;
    }
  }

  /**
   * Get total document count across all collections
   */
  _getTotalDocumentCount() {
    return Object.values(this.database.collections).reduce(
      (total, collection) => total + collection.length,
      0
    );
  }

  // ==========================================
  // KEY-VALUE STORAGE OPERATIONS
  // ==========================================

  /**
   * Store a key-value pair
   */
  async set(key, value) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    if (!key || typeof key !== "string") {
      throw new Error("Key must be a non-empty string");
    }

    try {
      this.database.keyValueStore[key] = value;
      await this._saveDatabase();

      this.log(`✓ Set key-value: ${key}`);
      return true;
    } catch (error) {
      this.log(`❌ Failed to set key-value: ${key}`, error);
      throw error;
    }
  }

  /**
   * Get a value by key
   */
  async get(key) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    if (!key || typeof key !== "string") {
      throw new Error("Key must be a non-empty string");
    }

    try {
      const value = this.database.keyValueStore[key];
      const result = value !== undefined ? value : null;

      this.log(
        `✓ Get key-value: ${key} => ${result !== null ? "found" : "null"}`
      );
      return result;
    } catch (error) {
      this.log(`❌ Failed to get key-value: ${key}`, error);
      throw error;
    }
  }

  /**
   * Remove a key-value pair
   */
  async remove(key) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    if (!key || typeof key !== "string") {
      throw new Error("Key must be a non-empty string");
    }

    try {
      const existed = this.database.keyValueStore.hasOwnProperty(key);
      delete this.database.keyValueStore[key];

      if (existed) {
        await this._saveDatabase();
      }

      this.log(`✓ Removed key-value: ${key}`);
      return existed;
    } catch (error) {
      this.log(`❌ Failed to remove key-value: ${key}`, error);
      throw error;
    }
  }

  /**
   * Get all key-value keys
   */
  async getAllKeys() {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    return Object.keys(this.database.keyValueStore);
  }

  /**
   * Get all key-value pairs
   */
  async getAll() {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    return { ...this.database.keyValueStore };
  }

  /**
   * Check if a key exists
   */
  async has(key) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    if (!key || typeof key !== "string") {
      throw new Error("Key must be a non-empty string");
    }

    return this.database.keyValueStore.hasOwnProperty(key);
  }

  // ==========================================
  // COLLECTION OPERATIONS
  // ==========================================

  /**
   * Get or create a collection
   */
  collection(name) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    if (!name || typeof name !== "string") {
      throw new Error("Collection name must be a non-empty string");
    }

    // Check collection limit
    const existingCollectionCount = Object.keys(
      this.database.collections
    ).length;
    if (
      existingCollectionCount >= this.options.maxCollections &&
      !this.database.collections[name]
    ) {
      throw new Error(
        `Maximum number of collections (${this.options.maxCollections}) exceeded`
      );
    }

    // Initialize collection in database if it doesn't exist
    if (!this.database.collections[name]) {
      this.database.collections[name] = [];
      this.database.indexes[name] = {};
    }

    // Create and return collection instance
    const collection = new Collection(name, this);

    this.log(
      `📁 Accessed collection: ${name} (${this.database.collections[name].length} documents)`
    );
    return collection;
  }

  /**
   * Get all collection names
   */
  getCollectionNames() {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    return Object.keys(this.database.collections);
  }

  /**
   * Drop/delete a collection
   */
  async dropCollection(name) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    try {
      if (this.database.collections[name]) {
        delete this.database.collections[name];
        delete this.database.indexes[name];

        await this._saveDatabase();

        this.log(`🗑️ Dropped collection: ${name}`);
        return true;
      }

      return false;
    } catch (error) {
      this.log(`❌ Failed to drop collection: ${name}`, error);
      throw error;
    }
  }

  // ==========================================
  // INTERNAL METHODS (used by Collection class)
  // ==========================================

  /**
   * This method is no longer needed as we save the entire database as one file
   * Keeping for backward compatibility but it now triggers a full database save
   */
  async _saveCollection(name, data) {
    if (!this.options.autoSave) {
      return;
    }

    try {
      // Check document limit
      if (data && data.length > this.options.maxDocuments) {
        throw new Error(
          `Collection ${name} exceeds maximum documents (${this.options.maxDocuments})`
        );
      }

      // Save the entire database since we're using file-based storage
      await this._saveDatabase();

      this.log(`💾 Collection ${name} saved as part of database file`);
    } catch (error) {
      this.log(`❌ Failed to save collection: ${name}`, error);
      throw error;
    }
  }

  // ==========================================
  // UTILITY METHODS
  // ==========================================

  /**
   * Get storage information
   */
  async getInfo() {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    const totalDocuments = Object.values(this.database.collections).reduce(
      (sum, docs) => sum + docs.length,
      0
    );

    return {
      initialized: this.initialized,
      collections: {
        count: Object.keys(this.database.collections).length,
        names: Object.keys(this.database.collections),
        totalDocuments,
      },
      keyValuePairs: Object.keys(this.database.keyValueStore).length,
      options: this.options,
      databasePath: this.databasePath,
    };
  }

  /**
   * Clear all data
   */
  async clear() {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    try {
      // Reset database structure
      const now = new Date().toISOString();
      this.database = {
        metadata: {
          version: "2.0.0",
          created: this.database.metadata.created || now,
          lastModified: now,
          collections: [],
          schema: {},
        },
        collections: {},
        indexes: {},
        keyValueStore: {},
      };

      // Save empty database to file
      await this._saveDatabase();

      this.log("🗑️ Cleared all SSTStorage data");
      return true;
    } catch (error) {
      this.log("❌ Failed to clear SSTStorage:", error);
      throw error;
    }
  }

  /**
   * Export all data (for backup)
   */
  async exportData() {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    const exportData = {
      timestamp: new Date().toISOString(),
      version: "2.0.0",
      metadata: this.database.metadata,
      collections: this.database.collections,
      keyValueStore: this.database.keyValueStore,
      indexes: this.database.indexes,
    };

    this.log("📤 Exported SSTStorage data");
    return exportData;
  }

  /**
   * Import data (for restore)
   */
  async importData(importData) {
    if (!this.initialized) {
      throw new Error("SSTStorage not initialized. Call initialize() first.");
    }

    try {
      // Validate import data structure
      if (!importData || typeof importData !== "object") {
        throw new Error("Invalid import data");
      }

      // Import collections
      if (importData.collections) {
        this.database.collections = importData.collections;
      }

      // Import key-value data
      if (importData.keyValueStore) {
        this.database.keyValueStore = importData.keyValueStore;
      }

      // Import indexes if available
      if (importData.indexes) {
        this.database.indexes = importData.indexes;
      }

      // Import metadata if available
      if (importData.metadata) {
        this.database.metadata = {
          ...this.database.metadata,
          ...importData.metadata,
          lastModified: new Date().toISOString(),
        };
      }

      // Update collection list in metadata
      this.database.metadata.collections = Object.keys(
        this.database.collections
      );

      // Save imported data to file
      await this._saveDatabase();

      this.log("📥 Imported SSTStorage data successfully");
      return true;
    } catch (error) {
      this.log("❌ Failed to import SSTStorage data:", error);
      throw error;
    }
  }
}

module.exports = SSTStorage;