appwrite-utils-cli/src/migrations/transfer.ts

Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.

import { converterFunctions, tryAwaitWithRetry } from "appwrite-utils";
import {
  Client,
  Databases,
  IndexType,
  Query,
  Storage,
  Users,
  type Models,
} from "node-appwrite";
import { InputFile } from "node-appwrite/file";
import { getAppwriteClient } from "../utils/helperFunctions.js";
import {
  createOrUpdateAttribute,
  createUpdateCollectionAttributes,
  createUpdateCollectionAttributesWithStatusCheck,
} from "../collections/attributes.js";
import { parseAttribute } from "appwrite-utils";
import chalk from "chalk";
import { fetchAllCollections } from "../collections/methods.js";
import { MessageFormatter } from "../shared/messageFormatter.js";
import { ProgressManager } from "../shared/progressManager.js";
import {
  createOrUpdateIndex,
  createOrUpdateIndexes,
  createOrUpdateIndexesWithStatusCheck,
} from "../collections/indexes.js";
import { getClient } from "../utils/getClientFromConfig.js";

export interface TransferOptions {
  fromDb: Models.Database | undefined;
  targetDb: Models.Database | undefined;
  isRemote: boolean;
  collections?: string[];
  transferEndpoint?: string;
  transferProject?: string;
  transferKey?: string;
  sourceBucket?: Models.Bucket;
  targetBucket?: Models.Bucket;
  transferUsers?: boolean;
}

export const transferStorageLocalToLocal = async (
  storage: Storage,
  fromBucketId: string,
  toBucketId: string
) => {
  MessageFormatter.info(`Transferring files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
  let lastFileId: string | undefined;
  let fromFiles = await tryAwaitWithRetry(
    async () => await storage.listFiles(fromBucketId, [Query.limit(100)])
  );
  const allFromFiles = fromFiles.files;
  let numberOfFiles = 0;

  const downloadFileWithRetry = async (bucketId: string, fileId: string) => {
    let attempts = 3;
    while (attempts > 0) {
      try {
        return await storage.getFileDownload(bucketId, fileId);
      } catch (error) {
        MessageFormatter.error(`Error downloading file ${fileId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
        attempts--;
        if (attempts === 0) throw error;
      }
    }
  };

  if (fromFiles.files.length < 100) {
    for (const file of allFromFiles) {
      const fileData = await tryAwaitWithRetry(
        async () => await downloadFileWithRetry(file.bucketId, file.$id)
      );
      if (!fileData) {
        MessageFormatter.error(`Error downloading file ${file.$id}`, undefined, { prefix: "Transfer" });
        continue;
      }
      const fileToCreate = InputFile.fromBuffer(
        new Uint8Array(fileData),
        file.name
      );
      MessageFormatter.progress(`Creating file: ${file.name}`, { prefix: "Transfer" });
      try {
        await tryAwaitWithRetry(
          async () =>
            await storage.createFile(
              toBucketId,
              file.$id,
              fileToCreate,
              file.$permissions
            )
        );
      } catch (error: any) {
        // File already exists, so we can skip it
        continue;
      }
      numberOfFiles++;
    }
  } else {
    lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
    while (lastFileId) {
      const files = await tryAwaitWithRetry(
        async () =>
          await storage.listFiles(fromBucketId, [
            Query.limit(100),
            Query.cursorAfter(lastFileId!),
          ])
      );
      allFromFiles.push(...files.files);
      if (files.files.length < 100) {
        lastFileId = undefined;
      } else {
        lastFileId = files.files[files.files.length - 1].$id;
      }
    }
    for (const file of allFromFiles) {
      const fileData = await tryAwaitWithRetry(
        async () => await downloadFileWithRetry(file.bucketId, file.$id)
      );
      if (!fileData) {
        MessageFormatter.error(`Error downloading file ${file.$id}`, undefined, { prefix: "Transfer" });
        continue;
      }
      const fileToCreate = InputFile.fromBuffer(
        new Uint8Array(fileData),
        file.name
      );
      try {
        await tryAwaitWithRetry(
          async () =>
            await storage.createFile(
              toBucketId,
              file.$id,
              fileToCreate,
              file.$permissions
            )
        );
      } catch (error: any) {
        // File already exists, so we can skip it
        MessageFormatter.warning(`File ${file.$id} already exists, skipping...`, { prefix: "Transfer" });
        continue;
      }
      numberOfFiles++;
    }
  }

  MessageFormatter.success(
    `Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`,
    { prefix: "Transfer" }
  );
};

export const transferStorageLocalToRemote = async (
  localStorage: Storage,
  endpoint: string,
  projectId: string,
  apiKey: string,
  fromBucketId: string,
  toBucketId: string
) => {
  MessageFormatter.info(
    `Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`,
    { prefix: "Transfer" }
  );
  const client = getAppwriteClient(endpoint, projectId, apiKey);
  const remoteStorage = new Storage(client);
  let numberOfFiles = 0;
  let lastFileId: string | undefined;
  let fromFiles = await tryAwaitWithRetry(
    async () => await localStorage.listFiles(fromBucketId, [Query.limit(100)])
  );
  const allFromFiles = fromFiles.files;
  if (fromFiles.files.length === 100) {
    lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
    while (lastFileId) {
      const files = await tryAwaitWithRetry(
        async () =>
          await localStorage.listFiles(fromBucketId, [
            Query.limit(100),
            Query.cursorAfter(lastFileId!),
          ])
      );
      allFromFiles.push(...files.files);
      if (files.files.length < 100) {
        break;
      }
      lastFileId = files.files[files.files.length - 1].$id;
    }
  }

  for (const file of allFromFiles) {
    const fileData = await tryAwaitWithRetry(
      async () => await localStorage.getFileDownload(file.bucketId, file.$id)
    );
    const fileToCreate = InputFile.fromBuffer(
      new Uint8Array(fileData),
      file.name
    );
    try {
      await tryAwaitWithRetry(
        async () =>
          await remoteStorage.createFile(
            toBucketId,
            file.$id,
            fileToCreate,
            file.$permissions
          )
      );
    } catch (error: any) {
      // File already exists, so we can skip it
      MessageFormatter.warning(`File ${file.$id} already exists, skipping...`, { prefix: "Transfer" });
      continue;
    }
    numberOfFiles++;
  }
  MessageFormatter.success(
    `Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`,
    { prefix: "Transfer" }
  );
};

// Document transfer functions moved to collections/methods.ts with enhanced UX

// Remote document transfer functions moved to collections/methods.ts with enhanced UX

/**
 * Transfers all collections and documents from one local database to another local database.
 *
 * @param {Databases} localDb - The local database instance.
 * @param {string} fromDbId - The ID of the source database.
 * @param {string} targetDbId - The ID of the target database.
 * @return {Promise<void>} A promise that resolves when the transfer is complete.
 */
export const transferDatabaseLocalToLocal = async (
  localDb: Databases,
  fromDbId: string,
  targetDbId: string
) => {
  console.log(
    chalk.blue(`Starting database transfer from ${fromDbId} to ${targetDbId}`)
  );
  // Get all collections from source database
  const sourceCollections = await fetchAllCollections(fromDbId, localDb);
  console.log(
    chalk.blue(
      `Found ${sourceCollections.length} collections in source database`
    )
  );

  // Process each collection
  for (const collection of sourceCollections) {
    console.log(
      chalk.yellow(
        `Processing collection: ${collection.name} (${collection.$id})`
      )
    );

    try {
      // Create or update collection in target
      let targetCollection: Models.Collection;
      const existingCollection = await tryAwaitWithRetry(async () =>
        localDb.listCollections(targetDbId, [
          Query.equal("$id", collection.$id),
        ])
      );

      if (existingCollection.collections.length > 0) {
        targetCollection = existingCollection.collections[0];
        console.log(
          chalk.green(`Collection ${collection.name} exists in target database`)
        );

        // Update collection if needed
        if (
          targetCollection.name !== collection.name ||
          targetCollection.$permissions !== collection.$permissions ||
          targetCollection.documentSecurity !== collection.documentSecurity ||
          targetCollection.enabled !== collection.enabled
        ) {
          targetCollection = await tryAwaitWithRetry(async () =>
            localDb.updateCollection(
              targetDbId,
              collection.$id,
              collection.name,
              collection.$permissions,
              collection.documentSecurity,
              collection.enabled
            )
          );
          console.log(chalk.green(`Collection ${collection.name} updated`));
        }
      } else {
        console.log(
          chalk.yellow(
            `Creating collection ${collection.name} in target database...`
          )
        );
        targetCollection = await tryAwaitWithRetry(async () =>
          localDb.createCollection(
            targetDbId,
            collection.$id,
            collection.name,
            collection.$permissions,
            collection.documentSecurity,
            collection.enabled
          )
        );
      }

      // Handle attributes with enhanced status checking
      console.log(chalk.blue(`Creating attributes for collection ${collection.name} with enhanced monitoring...`));
      
      const allAttributes = collection.attributes.map(attr => parseAttribute(attr as any));
      const attributeSuccess = await createUpdateCollectionAttributesWithStatusCheck(
        localDb,
        targetDbId,
        targetCollection,
        allAttributes
      );
      
      if (!attributeSuccess) {
        console.log(chalk.red(`❌ Failed to create all attributes for collection ${collection.name}, skipping to next collection`));
        continue;
      }
      
      console.log(chalk.green(`✅ All attributes created successfully for collection ${collection.name}`));

      // Handle indexes
      const existingIndexes = await tryAwaitWithRetry(
        async () => await localDb.listIndexes(targetDbId, targetCollection.$id)
      );

      for (const index of collection.indexes) {
        const existingIndex = existingIndexes.indexes.find(
          (idx) => idx.key === index.key
        );

        if (!existingIndex) {
          await tryAwaitWithRetry(async () =>
            createOrUpdateIndex(
              targetDbId,
              localDb,
              targetCollection.$id,
              index as any
            )
          );
          console.log(chalk.green(`Index ${index.key} created`));
        } else {
          console.log(
            chalk.blue(`Index ${index.key} exists, checking for updates...`)
          );
          await tryAwaitWithRetry(async () =>
            createOrUpdateIndex(
              targetDbId,
              localDb,
              targetCollection.$id,
              index as any
            )
          );
        }
      }

      // Transfer documents
      const { transferDocumentsBetweenDbsLocalToLocal } = await import("../collections/methods.js");
      await transferDocumentsBetweenDbsLocalToLocal(
        localDb,
        fromDbId,
        targetDbId,
        collection.$id,
        targetCollection.$id
      );
    } catch (error) {
      console.error(
        chalk.red(`Error processing collection ${collection.name}:`),
        error
      );
    }
  }
};

export const transferDatabaseLocalToRemote = async (
  localDb: Databases,
  endpoint: string,
  projectId: string,
  apiKey: string,
  fromDbId: string,
  toDbId: string
) => {
  const client = getAppwriteClient(endpoint, projectId, apiKey);
  const remoteDb = new Databases(client);

  // Get all collections from source database
  const sourceCollections = await fetchAllCollections(fromDbId, localDb);
  console.log(
    chalk.blue(
      `Found ${sourceCollections.length} collections in source database`
    )
  );

  // Process each collection
  for (const collection of sourceCollections) {
    console.log(
      chalk.yellow(
        `Processing collection: ${collection.name} (${collection.$id})`
      )
    );

    try {
      // Create or update collection in target
      let targetCollection: Models.Collection;
      const existingCollection = await tryAwaitWithRetry(async () =>
        remoteDb.listCollections(toDbId, [Query.equal("$id", collection.$id)])
      );

      if (existingCollection.collections.length > 0) {
        targetCollection = existingCollection.collections[0];
        console.log(
          chalk.green(`Collection ${collection.name} exists in remote database`)
        );

        // Update collection if needed
        if (
          targetCollection.name !== collection.name ||
          targetCollection.$permissions !== collection.$permissions ||
          targetCollection.documentSecurity !== collection.documentSecurity ||
          targetCollection.enabled !== collection.enabled
        ) {
          targetCollection = await tryAwaitWithRetry(async () =>
            remoteDb.updateCollection(
              toDbId,
              collection.$id,
              collection.name,
              collection.$permissions,
              collection.documentSecurity,
              collection.enabled
            )
          );
          console.log(chalk.green(`Collection ${collection.name} updated`));
        }
      } else {
        console.log(
          chalk.yellow(
            `Creating collection ${collection.name} in remote database...`
          )
        );
        targetCollection = await tryAwaitWithRetry(async () =>
          remoteDb.createCollection(
            toDbId,
            collection.$id,
            collection.name,
            collection.$permissions,
            collection.documentSecurity,
            collection.enabled
          )
        );
      }

      // Handle attributes with enhanced status checking
      console.log(chalk.blue(`Creating attributes for collection ${collection.name} with enhanced monitoring...`));
      
      const attributesToCreate = collection.attributes.map(attr => parseAttribute(attr as any));
      
      const attributesSuccess = await createUpdateCollectionAttributesWithStatusCheck(
        remoteDb,
        toDbId,
        targetCollection,
        attributesToCreate
      );
      
      if (!attributesSuccess) {
        console.log(chalk.red(`Failed to create some attributes for collection ${collection.name}`));
        // Continue with the transfer even if some attributes failed
      } else {
        console.log(chalk.green(`All attributes created successfully for collection ${collection.name}`));
      }

      // Handle indexes with enhanced status checking
      console.log(chalk.blue(`Creating indexes for collection ${collection.name} with enhanced monitoring...`));
      
      const indexesSuccess = await createOrUpdateIndexesWithStatusCheck(
        toDbId,
        remoteDb,
        targetCollection.$id,
        targetCollection,
        collection.indexes as any
      );
      
      if (!indexesSuccess) {
        console.log(chalk.red(`Failed to create some indexes for collection ${collection.name}`));
        // Continue with the transfer even if some indexes failed
      } else {
        console.log(chalk.green(`All indexes created successfully for collection ${collection.name}`));
      }

      // Transfer documents
      const { transferDocumentsBetweenDbsLocalToRemote } = await import("../collections/methods.js");
      await transferDocumentsBetweenDbsLocalToRemote(
        localDb,
        endpoint,
        projectId,
        apiKey,
        fromDbId,
        toDbId,
        collection.$id,
        targetCollection.$id
      );
    } catch (error) {
      console.error(
        chalk.red(`Error processing collection ${collection.name}:`),
        error
      );
    }
  }
};

export const transferUsersLocalToRemote = async (
  localUsers: Users,
  endpoint: string,
  projectId: string,
  apiKey: string
) => {
  console.log(chalk.blue("Starting user transfer to remote instance..."));

  const client = getClient(endpoint, projectId, apiKey);
  const remoteUsers = new Users(client);

  let totalTransferred = 0;
  let lastId: string | undefined;

  while (true) {
    const queries = [Query.limit(100)];
    if (lastId) {
      queries.push(Query.cursorAfter(lastId));
    }

    const usersList = await tryAwaitWithRetry(async () =>
      localUsers.list(queries)
    );

    if (usersList.users.length === 0) {
      break;
    }

    for (const user of usersList.users) {
      try {
        // Check if user already exists in remote
        try {
          await tryAwaitWithRetry(async () => remoteUsers.get(user.$id));
          console.log(
            chalk.yellow(`User ${user.$id} already exists, skipping...`)
          );
          continue;
        } catch (error: any) {
          // User doesn't exist, proceed with creation
        }

        const phone = user.phone
          ? converterFunctions.convertPhoneStringToUSInternational(user.phone)
          : undefined;

        // Handle user creation based on hash type
        if (user.hash && user.password) {
          // User has a hashed password - recreate with proper hash method
          const hashType = user.hash.toLowerCase();
          const hashedPassword = user.password; // This is already hashed
          const hashOptions = (user.hashOptions as Record<string, any>) || {};
          
          try {
            switch (hashType) {
              case 'argon2':
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createArgon2User(
                    user.$id,
                    user.email,
                    hashedPassword,
                    user.name
                  )
                );
                break;
                
              case 'bcrypt':
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createBcryptUser(
                    user.$id,
                    user.email,
                    hashedPassword,
                    user.name
                  )
                );
                break;
                
              case 'scrypt':
                // Scrypt requires additional parameters from hashOptions
                const salt = typeof hashOptions.salt === 'string' ? hashOptions.salt : '';
                const costCpu = typeof hashOptions.costCpu === 'number' ? hashOptions.costCpu : 32768;
                const costMemory = typeof hashOptions.costMemory === 'number' ? hashOptions.costMemory : 14;
                const costParallel = typeof hashOptions.costParallel === 'number' ? hashOptions.costParallel : 1;
                const length = typeof hashOptions.length === 'number' ? hashOptions.length : 64;
                
                // Warn if using default values due to missing hash options
                if (!hashOptions.salt || typeof hashOptions.costCpu !== 'number') {
                  console.log(chalk.yellow(`User ${user.$id}: Using default Scrypt parameters due to missing hashOptions`));
                }
                
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createScryptUser(
                    user.$id,
                    user.email,
                    hashedPassword,
                    salt,
                    costCpu,
                    costMemory,
                    costParallel,
                    length,
                    user.name
                  )
                );
                break;
                
              case 'scryptmodified':
                // Scrypt Modified (Firebase) requires salt, separator, and signer key
                const modSalt = typeof hashOptions.salt === 'string' ? hashOptions.salt : '';
                const saltSeparator = typeof hashOptions.saltSeparator === 'string' ? hashOptions.saltSeparator : '';
                const signerKey = typeof hashOptions.signerKey === 'string' ? hashOptions.signerKey : '';
                
                // Warn if critical parameters are missing
                if (!hashOptions.salt || !hashOptions.saltSeparator || !hashOptions.signerKey) {
                  console.log(chalk.yellow(`User ${user.$id}: Missing critical Scrypt Modified parameters in hashOptions`));
                }
                
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createScryptModifiedUser(
                    user.$id,
                    user.email,
                    hashedPassword,
                    modSalt,
                    saltSeparator,
                    signerKey,
                    user.name
                  )
                );
                break;
                
              case 'md5':
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createMD5User(
                    user.$id,
                    user.email,
                    hashedPassword,
                    user.name
                  )
                );
                break;
                
              case 'sha':
              case 'sha1':
              case 'sha256':
              case 'sha512':
                // SHA variants - determine version from hash type
                const getPasswordHashVersion = (hash: string) => {
                  switch (hash.toLowerCase()) {
                    case 'sha1': return 'sha1' as any;
                    case 'sha256': return 'sha256' as any;
                    case 'sha512': return 'sha512' as any;
                    default: return 'sha256' as any; // Default to SHA256
                  }
                };
                
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createSHAUser(
                    user.$id,
                    user.email,
                    hashedPassword,
                    getPasswordHashVersion(hashType),
                    user.name
                  )
                );
                break;
                
              case 'phpass':
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createPHPassUser(
                    user.$id,
                    user.email,
                    hashedPassword,
                    user.name
                  )
                );
                break;
                
              default:
                console.log(chalk.yellow(`Unknown hash type '${hashType}' for user ${user.$id}, falling back to Argon2`));
                await tryAwaitWithRetry(async () =>
                  remoteUsers.createArgon2User(
                    user.$id,
                    user.email,
                    hashedPassword,
                    user.name
                  )
                );
                break;
            }
            
            console.log(chalk.green(`User ${user.$id} created with preserved ${hashType} password`));
            
          } catch (error) {
            console.log(chalk.yellow(`Failed to create user ${user.$id} with ${hashType} hash, trying with temporary password`));
            
            // Fallback to creating user with temporary password
            await tryAwaitWithRetry(async () =>
              remoteUsers.create(
                user.$id,
                user.email,
                phone,
                `changeMe${user.email}`,
                user.name
              )
            );
            
            console.log(chalk.yellow(`User ${user.$id} created with temporary password - password reset required`));
          }
          
        } else {
          // No hash or password - create with temporary password
          const tempPassword = user.password || `changeMe${user.email}`;
          
          await tryAwaitWithRetry(async () =>
            remoteUsers.create(
              user.$id,
              user.email,
              phone,
              tempPassword,
              user.name
            )
          );
          
          if (!user.password) {
            console.log(chalk.yellow(`User ${user.$id} created with temporary password - password reset required`));
          }
        }
        
        // Update phone, labels, and other attributes
        if (phone) {
          await tryAwaitWithRetry(async () =>
            remoteUsers.updatePhone(user.$id, phone)
          );
        }
        
        if (user.labels && user.labels.length > 0) {
          await tryAwaitWithRetry(async () =>
            remoteUsers.updateLabels(user.$id, user.labels)
          );
        }

        // Update user preferences and status
        await tryAwaitWithRetry(async () =>
          remoteUsers.updatePrefs(user.$id, user.prefs)
        );

        if (!user.emailVerification) {
          await tryAwaitWithRetry(async () =>
            remoteUsers.updateEmailVerification(user.$id, false)
          );
        }

        if (user.status === false) {
          await tryAwaitWithRetry(async () =>
            remoteUsers.updateStatus(user.$id, false)
          );
        }

        totalTransferred++;
        console.log(chalk.green(`Transferred user ${user.$id}`));
      } catch (error) {
        console.error(chalk.red(`Failed to transfer user ${user.$id}:`), error);
      }
    }

    if (usersList.users.length < 100) {
      break;
    }
    lastId = usersList.users[usersList.users.length - 1].$id;
  }

  console.log(
    chalk.green(`Successfully transferred ${totalTransferred} users`)
  );
};