appwrite-utils-cli
Version:
Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.
393 lines (392 loc) • 24.2 kB
JavaScript
import { converterFunctions, tryAwaitWithRetry } from "appwrite-utils";
import { Client, Databases, IndexType, Query, Storage, Users, } from "node-appwrite";
import { InputFile } from "node-appwrite/file";
import { getAppwriteClient } from "../utils/helperFunctions.js";
import { createOrUpdateAttribute, createUpdateCollectionAttributes, createUpdateCollectionAttributesWithStatusCheck, } from "../collections/attributes.js";
import { parseAttribute } from "appwrite-utils";
import chalk from "chalk";
import { fetchAllCollections } from "../collections/methods.js";
import { MessageFormatter } from "../shared/messageFormatter.js";
import { ProgressManager } from "../shared/progressManager.js";
import { createOrUpdateIndex, createOrUpdateIndexes, createOrUpdateIndexesWithStatusCheck, } from "../collections/indexes.js";
import { getClient } from "../utils/getClientFromConfig.js";
export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucketId) => {
MessageFormatter.info(`Transferring files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
let lastFileId;
let fromFiles = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [Query.limit(100)]));
const allFromFiles = fromFiles.files;
let numberOfFiles = 0;
const downloadFileWithRetry = async (bucketId, fileId) => {
let attempts = 3;
while (attempts > 0) {
try {
return await storage.getFileDownload(bucketId, fileId);
}
catch (error) {
MessageFormatter.error(`Error downloading file ${fileId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
attempts--;
if (attempts === 0)
throw error;
}
}
};
if (fromFiles.files.length < 100) {
for (const file of allFromFiles) {
const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
if (!fileData) {
MessageFormatter.error(`Error downloading file ${file.$id}`, undefined, { prefix: "Transfer" });
continue;
}
const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
MessageFormatter.progress(`Creating file: ${file.name}`, { prefix: "Transfer" });
try {
await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
}
catch (error) {
// File already exists, so we can skip it
continue;
}
numberOfFiles++;
}
}
else {
lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
while (lastFileId) {
const files = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [
Query.limit(100),
Query.cursorAfter(lastFileId),
]));
allFromFiles.push(...files.files);
if (files.files.length < 100) {
lastFileId = undefined;
}
else {
lastFileId = files.files[files.files.length - 1].$id;
}
}
for (const file of allFromFiles) {
const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
if (!fileData) {
MessageFormatter.error(`Error downloading file ${file.$id}`, undefined, { prefix: "Transfer" });
continue;
}
const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
try {
await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
}
catch (error) {
// File already exists, so we can skip it
MessageFormatter.warning(`File ${file.$id} already exists, skipping...`, { prefix: "Transfer" });
continue;
}
numberOfFiles++;
}
}
MessageFormatter.success(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
};
export const transferStorageLocalToRemote = async (localStorage, endpoint, projectId, apiKey, fromBucketId, toBucketId) => {
MessageFormatter.info(`Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`, { prefix: "Transfer" });
const client = getAppwriteClient(endpoint, projectId, apiKey);
const remoteStorage = new Storage(client);
let numberOfFiles = 0;
let lastFileId;
let fromFiles = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [Query.limit(100)]));
const allFromFiles = fromFiles.files;
if (fromFiles.files.length === 100) {
lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
while (lastFileId) {
const files = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [
Query.limit(100),
Query.cursorAfter(lastFileId),
]));
allFromFiles.push(...files.files);
if (files.files.length < 100) {
break;
}
lastFileId = files.files[files.files.length - 1].$id;
}
}
for (const file of allFromFiles) {
const fileData = await tryAwaitWithRetry(async () => await localStorage.getFileDownload(file.bucketId, file.$id));
const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
try {
await tryAwaitWithRetry(async () => await remoteStorage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
}
catch (error) {
// File already exists, so we can skip it
MessageFormatter.warning(`File ${file.$id} already exists, skipping...`, { prefix: "Transfer" });
continue;
}
numberOfFiles++;
}
MessageFormatter.success(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`, { prefix: "Transfer" });
};
// Document transfer functions moved to collections/methods.ts with enhanced UX
// Remote document transfer functions moved to collections/methods.ts with enhanced UX
/**
* Transfers all collections and documents from one local database to another local database.
*
* @param {Databases} localDb - The local database instance.
* @param {string} fromDbId - The ID of the source database.
* @param {string} targetDbId - The ID of the target database.
* @return {Promise<void>} A promise that resolves when the transfer is complete.
*/
export const transferDatabaseLocalToLocal = async (localDb, fromDbId, targetDbId) => {
console.log(chalk.blue(`Starting database transfer from ${fromDbId} to ${targetDbId}`));
// Get all collections from source database
const sourceCollections = await fetchAllCollections(fromDbId, localDb);
console.log(chalk.blue(`Found ${sourceCollections.length} collections in source database`));
// Process each collection
for (const collection of sourceCollections) {
console.log(chalk.yellow(`Processing collection: ${collection.name} (${collection.$id})`));
try {
// Create or update collection in target
let targetCollection;
const existingCollection = await tryAwaitWithRetry(async () => localDb.listCollections(targetDbId, [
Query.equal("$id", collection.$id),
]));
if (existingCollection.collections.length > 0) {
targetCollection = existingCollection.collections[0];
console.log(chalk.green(`Collection ${collection.name} exists in target database`));
// Update collection if needed
if (targetCollection.name !== collection.name ||
targetCollection.$permissions !== collection.$permissions ||
targetCollection.documentSecurity !== collection.documentSecurity ||
targetCollection.enabled !== collection.enabled) {
targetCollection = await tryAwaitWithRetry(async () => localDb.updateCollection(targetDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
console.log(chalk.green(`Collection ${collection.name} updated`));
}
}
else {
console.log(chalk.yellow(`Creating collection ${collection.name} in target database...`));
targetCollection = await tryAwaitWithRetry(async () => localDb.createCollection(targetDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
}
// Handle attributes with enhanced status checking
console.log(chalk.blue(`Creating attributes for collection ${collection.name} with enhanced monitoring...`));
const allAttributes = collection.attributes.map(attr => parseAttribute(attr));
const attributeSuccess = await createUpdateCollectionAttributesWithStatusCheck(localDb, targetDbId, targetCollection, allAttributes);
if (!attributeSuccess) {
console.log(chalk.red(`❌ Failed to create all attributes for collection ${collection.name}, skipping to next collection`));
continue;
}
console.log(chalk.green(`✅ All attributes created successfully for collection ${collection.name}`));
// Handle indexes
const existingIndexes = await tryAwaitWithRetry(async () => await localDb.listIndexes(targetDbId, targetCollection.$id));
for (const index of collection.indexes) {
const existingIndex = existingIndexes.indexes.find((idx) => idx.key === index.key);
if (!existingIndex) {
await tryAwaitWithRetry(async () => createOrUpdateIndex(targetDbId, localDb, targetCollection.$id, index));
console.log(chalk.green(`Index ${index.key} created`));
}
else {
console.log(chalk.blue(`Index ${index.key} exists, checking for updates...`));
await tryAwaitWithRetry(async () => createOrUpdateIndex(targetDbId, localDb, targetCollection.$id, index));
}
}
// Transfer documents
const { transferDocumentsBetweenDbsLocalToLocal } = await import("../collections/methods.js");
await transferDocumentsBetweenDbsLocalToLocal(localDb, fromDbId, targetDbId, collection.$id, targetCollection.$id);
}
catch (error) {
console.error(chalk.red(`Error processing collection ${collection.name}:`), error);
}
}
};
export const transferDatabaseLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId) => {
const client = getAppwriteClient(endpoint, projectId, apiKey);
const remoteDb = new Databases(client);
// Get all collections from source database
const sourceCollections = await fetchAllCollections(fromDbId, localDb);
console.log(chalk.blue(`Found ${sourceCollections.length} collections in source database`));
// Process each collection
for (const collection of sourceCollections) {
console.log(chalk.yellow(`Processing collection: ${collection.name} (${collection.$id})`));
try {
// Create or update collection in target
let targetCollection;
const existingCollection = await tryAwaitWithRetry(async () => remoteDb.listCollections(toDbId, [Query.equal("$id", collection.$id)]));
if (existingCollection.collections.length > 0) {
targetCollection = existingCollection.collections[0];
console.log(chalk.green(`Collection ${collection.name} exists in remote database`));
// Update collection if needed
if (targetCollection.name !== collection.name ||
targetCollection.$permissions !== collection.$permissions ||
targetCollection.documentSecurity !== collection.documentSecurity ||
targetCollection.enabled !== collection.enabled) {
targetCollection = await tryAwaitWithRetry(async () => remoteDb.updateCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
console.log(chalk.green(`Collection ${collection.name} updated`));
}
}
else {
console.log(chalk.yellow(`Creating collection ${collection.name} in remote database...`));
targetCollection = await tryAwaitWithRetry(async () => remoteDb.createCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
}
// Handle attributes with enhanced status checking
console.log(chalk.blue(`Creating attributes for collection ${collection.name} with enhanced monitoring...`));
const attributesToCreate = collection.attributes.map(attr => parseAttribute(attr));
const attributesSuccess = await createUpdateCollectionAttributesWithStatusCheck(remoteDb, toDbId, targetCollection, attributesToCreate);
if (!attributesSuccess) {
console.log(chalk.red(`Failed to create some attributes for collection ${collection.name}`));
// Continue with the transfer even if some attributes failed
}
else {
console.log(chalk.green(`All attributes created successfully for collection ${collection.name}`));
}
// Handle indexes with enhanced status checking
console.log(chalk.blue(`Creating indexes for collection ${collection.name} with enhanced monitoring...`));
const indexesSuccess = await createOrUpdateIndexesWithStatusCheck(toDbId, remoteDb, targetCollection.$id, targetCollection, collection.indexes);
if (!indexesSuccess) {
console.log(chalk.red(`Failed to create some indexes for collection ${collection.name}`));
// Continue with the transfer even if some indexes failed
}
else {
console.log(chalk.green(`All indexes created successfully for collection ${collection.name}`));
}
// Transfer documents
const { transferDocumentsBetweenDbsLocalToRemote } = await import("../collections/methods.js");
await transferDocumentsBetweenDbsLocalToRemote(localDb, endpoint, projectId, apiKey, fromDbId, toDbId, collection.$id, targetCollection.$id);
}
catch (error) {
console.error(chalk.red(`Error processing collection ${collection.name}:`), error);
}
}
};
export const transferUsersLocalToRemote = async (localUsers, endpoint, projectId, apiKey) => {
console.log(chalk.blue("Starting user transfer to remote instance..."));
const client = getClient(endpoint, projectId, apiKey);
const remoteUsers = new Users(client);
let totalTransferred = 0;
let lastId;
while (true) {
const queries = [Query.limit(100)];
if (lastId) {
queries.push(Query.cursorAfter(lastId));
}
const usersList = await tryAwaitWithRetry(async () => localUsers.list(queries));
if (usersList.users.length === 0) {
break;
}
for (const user of usersList.users) {
try {
// Check if user already exists in remote
try {
await tryAwaitWithRetry(async () => remoteUsers.get(user.$id));
console.log(chalk.yellow(`User ${user.$id} already exists, skipping...`));
continue;
}
catch (error) {
// User doesn't exist, proceed with creation
}
const phone = user.phone
? converterFunctions.convertPhoneStringToUSInternational(user.phone)
: undefined;
// Handle user creation based on hash type
if (user.hash && user.password) {
// User has a hashed password - recreate with proper hash method
const hashType = user.hash.toLowerCase();
const hashedPassword = user.password; // This is already hashed
const hashOptions = user.hashOptions || {};
try {
switch (hashType) {
case 'argon2':
await tryAwaitWithRetry(async () => remoteUsers.createArgon2User(user.$id, user.email, hashedPassword, user.name));
break;
case 'bcrypt':
await tryAwaitWithRetry(async () => remoteUsers.createBcryptUser(user.$id, user.email, hashedPassword, user.name));
break;
case 'scrypt':
// Scrypt requires additional parameters from hashOptions
const salt = typeof hashOptions.salt === 'string' ? hashOptions.salt : '';
const costCpu = typeof hashOptions.costCpu === 'number' ? hashOptions.costCpu : 32768;
const costMemory = typeof hashOptions.costMemory === 'number' ? hashOptions.costMemory : 14;
const costParallel = typeof hashOptions.costParallel === 'number' ? hashOptions.costParallel : 1;
const length = typeof hashOptions.length === 'number' ? hashOptions.length : 64;
// Warn if using default values due to missing hash options
if (!hashOptions.salt || typeof hashOptions.costCpu !== 'number') {
console.log(chalk.yellow(`User ${user.$id}: Using default Scrypt parameters due to missing hashOptions`));
}
await tryAwaitWithRetry(async () => remoteUsers.createScryptUser(user.$id, user.email, hashedPassword, salt, costCpu, costMemory, costParallel, length, user.name));
break;
case 'scryptmodified':
// Scrypt Modified (Firebase) requires salt, separator, and signer key
const modSalt = typeof hashOptions.salt === 'string' ? hashOptions.salt : '';
const saltSeparator = typeof hashOptions.saltSeparator === 'string' ? hashOptions.saltSeparator : '';
const signerKey = typeof hashOptions.signerKey === 'string' ? hashOptions.signerKey : '';
// Warn if critical parameters are missing
if (!hashOptions.salt || !hashOptions.saltSeparator || !hashOptions.signerKey) {
console.log(chalk.yellow(`User ${user.$id}: Missing critical Scrypt Modified parameters in hashOptions`));
}
await tryAwaitWithRetry(async () => remoteUsers.createScryptModifiedUser(user.$id, user.email, hashedPassword, modSalt, saltSeparator, signerKey, user.name));
break;
case 'md5':
await tryAwaitWithRetry(async () => remoteUsers.createMD5User(user.$id, user.email, hashedPassword, user.name));
break;
case 'sha':
case 'sha1':
case 'sha256':
case 'sha512':
// SHA variants - determine version from hash type
const getPasswordHashVersion = (hash) => {
switch (hash.toLowerCase()) {
case 'sha1': return 'sha1';
case 'sha256': return 'sha256';
case 'sha512': return 'sha512';
default: return 'sha256'; // Default to SHA256
}
};
await tryAwaitWithRetry(async () => remoteUsers.createSHAUser(user.$id, user.email, hashedPassword, getPasswordHashVersion(hashType), user.name));
break;
case 'phpass':
await tryAwaitWithRetry(async () => remoteUsers.createPHPassUser(user.$id, user.email, hashedPassword, user.name));
break;
default:
console.log(chalk.yellow(`Unknown hash type '${hashType}' for user ${user.$id}, falling back to Argon2`));
await tryAwaitWithRetry(async () => remoteUsers.createArgon2User(user.$id, user.email, hashedPassword, user.name));
break;
}
console.log(chalk.green(`User ${user.$id} created with preserved ${hashType} password`));
}
catch (error) {
console.log(chalk.yellow(`Failed to create user ${user.$id} with ${hashType} hash, trying with temporary password`));
// Fallback to creating user with temporary password
await tryAwaitWithRetry(async () => remoteUsers.create(user.$id, user.email, phone, `changeMe${user.email}`, user.name));
console.log(chalk.yellow(`User ${user.$id} created with temporary password - password reset required`));
}
}
else {
// No hash or password - create with temporary password
const tempPassword = user.password || `changeMe${user.email}`;
await tryAwaitWithRetry(async () => remoteUsers.create(user.$id, user.email, phone, tempPassword, user.name));
if (!user.password) {
console.log(chalk.yellow(`User ${user.$id} created with temporary password - password reset required`));
}
}
// Update phone, labels, and other attributes
if (phone) {
await tryAwaitWithRetry(async () => remoteUsers.updatePhone(user.$id, phone));
}
if (user.labels && user.labels.length > 0) {
await tryAwaitWithRetry(async () => remoteUsers.updateLabels(user.$id, user.labels));
}
// Update user preferences and status
await tryAwaitWithRetry(async () => remoteUsers.updatePrefs(user.$id, user.prefs));
if (!user.emailVerification) {
await tryAwaitWithRetry(async () => remoteUsers.updateEmailVerification(user.$id, false));
}
if (user.status === false) {
await tryAwaitWithRetry(async () => remoteUsers.updateStatus(user.$id, false));
}
totalTransferred++;
console.log(chalk.green(`Transferred user ${user.$id}`));
}
catch (error) {
console.error(chalk.red(`Failed to transfer user ${user.$id}:`), error);
}
}
if (usersList.users.length < 100) {
break;
}
lastId = usersList.users[usersList.users.length - 1].$id;
}
console.log(chalk.green(`Successfully transferred ${totalTransferred} users`));
};