appwrite-utils-cli
Version:
Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.
945 lines • 66.7 kB
JavaScript
import { AttributeMappingsSchema, CollectionCreateSchema, importDefSchema, } from "appwrite-utils";
import path from "path";
import fs from "fs";
import { convertObjectByAttributeMappings } from "../utils/dataConverters.js";
import { z } from "zod";
import { checkForCollection } from "../collections/methods.js";
import { ID, Users } from "node-appwrite";
import { logger } from "../shared/logging.js";
import { findOrCreateOperation, updateOperation } from "../shared/migrationHelpers.js";
import { AuthUserCreateSchema } from "../schemas/authUser.js";
import { LegacyAdapter } from "../adapters/LegacyAdapter.js";
import { UsersController } from "../users/methods.js";
import { finalizeByAttributeMap } from "../utils/helperFunctions.js";
import { isEmpty } from "es-toolkit/compat";
import { MessageFormatter } from "../shared/messageFormatter.js";
// Define a schema for the structure of collection import data using Zod for validation
export const CollectionImportDataSchema = z.object({
// Optional collection creation schema
collection: CollectionCreateSchema.optional(),
// Array of data objects each containing rawData, finalData, context, and an import definition
data: z.array(z.object({
rawData: z.any(), // The initial raw data
finalData: z.any(), // The transformed data ready for import
context: z.any(), // Additional context for the data transformation
importDef: importDefSchema.optional(), // The import definition schema
})),
});
// DataLoader class to handle the loading of data into collections
export class DataLoader {
// Private member variables to hold configuration and state
appwriteFolderPath;
importDataActions;
database;
usersController;
config;
// Map to hold the import data for each collection by name
importMap = new Map();
// Map to track old to new ID mappings for each collection, if applicable
oldIdToNewIdPerCollectionMap = new Map();
// Map to hold the import operation ID for each collection
collectionImportOperations = new Map();
// Map to hold the merged user map for relationship resolution
// Will hold an array of the old user ID's that are mapped to the same new user ID
// For example, if there are two users with the same email, they will both be mapped to the same new user ID
// Prevents duplicate users with the other two maps below it and allows me to keep the old ID's
mergedUserMap = new Map();
// Maps to hold email and phone to user ID mappings for unique-ness in User Accounts
emailToUserIdMap = new Map();
phoneToUserIdMap = new Map();
userIdSet = new Set();
userExistsMap = new Map();
shouldWriteFile = false;
// Constructor to initialize the DataLoader with necessary configurations
constructor(appwriteFolderPath, importDataActions, database, config, shouldWriteFile) {
this.appwriteFolderPath = appwriteFolderPath;
this.importDataActions = importDataActions;
this.database = database;
this.usersController = new UsersController(config, database);
this.config = config;
this.shouldWriteFile = shouldWriteFile || false;
}
// Helper method to generate a consistent key for collections
getCollectionKey(name) {
return name.toLowerCase().replace(" ", "");
}
/**
* Merges two objects by updating the source object with the target object's values.
* It iterates through the target object's keys and updates the source object if:
* - The source object has the key.
* - The target object's value for that key is not null, undefined, or an empty string.
* - If the target object has an array value, it concatenates the values and removes duplicates.
*
* @param source - The source object to be updated.
* @param target - The target object with values to update the source object.
* @returns The updated source object.
*/
mergeObjects(source, update) {
// Create a new object to hold the merged result
const result = { ...source };
// Loop through the keys of the object we care about
for (const [key, value] of Object.entries(source)) {
// Check if the key exists in the target object
if (!Object.hasOwn(update, key)) {
// If the key doesn't exist, we can just skip it like bad cheese
continue;
}
if (update[key] === value) {
continue;
}
// If the value ain't here, we can just do whatever man
if (value === undefined || value === null || value === "") {
// If the update key is defined
if (update[key] !== undefined &&
update[key] !== null &&
update[key] !== "") {
// might as well use it eh?
result[key] = update[key];
}
// ELSE if the value is an array, because it would then not be === to those things above
}
else if (Array.isArray(value)) {
// Get the update value
const updateValue = update[key];
// If the update value is an array, concatenate and remove duplicates
// and poopy data
if (Array.isArray(updateValue)) {
result[key] = [...new Set([...value, ...updateValue])].filter((item) => item !== null && item !== undefined && item !== "");
}
else {
// If the update value is not an array, just use it
result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
}
}
else if (typeof value === "object" && !Array.isArray(value)) {
// If the value is an object, we need to merge it
if (typeof update[key] === "object" && !Array.isArray(update[key])) {
result[key] = this.mergeObjects(value, update[key]);
}
}
else {
// Finally, the source value is defined, and not an array, so we don't care about the update value
continue;
}
}
// Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
for (const [key, value] of Object.entries(update)) {
if (value === undefined || value === null || value === "") {
continue;
}
else if (!Object.hasOwn(source, key)) {
result[key] = value;
}
else if (typeof source[key] === "object" &&
typeof value === "object" &&
!Array.isArray(source[key]) &&
!Array.isArray(value)) {
result[key] = this.mergeObjects(source[key], value);
}
else if (Array.isArray(source[key]) && Array.isArray(value)) {
result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
}
else if (source[key] === undefined ||
source[key] === null ||
source[key] === "") {
result[key] = value;
}
}
return result;
}
// Method to load data from a file specified in the import definition
loadData(importDef) {
// Simply join appwriteFolderPath with the importDef.filePath
const filePath = path.resolve(this.appwriteFolderPath, importDef.filePath);
MessageFormatter.info(`Loading data from: ${filePath}`, { prefix: "Data" });
if (!fs.existsSync(filePath)) {
MessageFormatter.error(`File not found: ${filePath}`, undefined, { prefix: "Data" });
return [];
}
// Read the file and parse the JSON data
const rawData = fs.readFileSync(filePath, "utf8");
const parsedData = importDef.basePath
? JSON.parse(rawData)[importDef.basePath]
: JSON.parse(rawData);
MessageFormatter.success(`Loaded ${parsedData?.length || 0} items from ${filePath}`, { prefix: "Data" });
return parsedData;
}
// Helper method to check if a new ID already exists in the old-to-new ID map
checkMapValuesForId(newId, collectionName) {
const oldIdMap = this.oldIdToNewIdPerCollectionMap.get(collectionName);
for (const [key, value] of oldIdMap?.entries() || []) {
if (value === newId) {
return key;
}
}
return false;
}
// Method to generate a unique ID that doesn't conflict with existing IDs
getTrueUniqueId(collectionName) {
let newId = ID.unique();
let condition = this.checkMapValuesForId(newId, collectionName) ||
this.userExistsMap.has(newId) ||
this.userIdSet.has(newId) ||
this.importMap
.get(this.getCollectionKey("users"))
?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
while (condition) {
newId = ID.unique();
condition =
this.checkMapValuesForId(newId, collectionName) ||
this.userExistsMap.has(newId) ||
this.userIdSet.has(newId) ||
this.importMap
.get(this.getCollectionKey("users"))
?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
}
return newId;
}
// Method to create a context object for data transformation
createContext(db, collection, item, docId) {
return {
...item, // Spread the item data for easy access to its properties
dbId: db.$id,
dbName: db.name,
collId: collection.$id,
collName: collection.name,
docId: docId,
createdDoc: {}, // Initially null, to be updated when the document is created
};
}
/**
* Transforms the given item based on the provided attribute mappings.
* This method applies conversion rules to the item's attributes as defined in the attribute mappings.
*
* @param item - The item to be transformed.
* @param attributeMappings - The mappings that define how each attribute should be transformed.
* @returns The transformed item.
*/
transformData(item, attributeMappings) {
// Convert the item using the attribute mappings provided
const convertedItem = convertObjectByAttributeMappings(item, attributeMappings);
// Run additional converter functions on the converted item, if any
return this.importDataActions.runConverterFunctions(convertedItem, attributeMappings);
}
async setupMaps(dbId) {
// Initialize the users collection in the import map
this.importMap.set(this.getCollectionKey("users"), {
data: [],
});
for (const db of this.config.databases) {
if (db.$id !== dbId) {
continue;
}
if (!this.config.collections) {
continue;
}
for (let index = 0; index < this.config.collections.length; index++) {
const collectionConfig = this.config.collections[index];
let collection = CollectionCreateSchema.parse(collectionConfig);
// Check if the collection exists in the database
const collectionExists = await checkForCollection(this.database, db.$id, collection);
if (!collectionExists) {
logger.error(`No collection found for ${collection.name}`);
continue;
}
else if (!collection.name) {
logger.error(`Collection ${collection.name} has no name`);
continue;
}
// Update the collection ID with the existing one
collectionConfig.$id = collectionExists.$id;
collection.$id = collectionExists.$id;
this.config.collections[index] = collectionConfig;
// Find or create an import operation for the collection
const adapter = new LegacyAdapter(this.database.client);
const collectionImportOperation = await findOrCreateOperation(adapter, dbId, "importData", collection.$id);
// Store the operation ID in the map
this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
// Initialize the collection in the import map
this.importMap.set(this.getCollectionKey(collection.name), {
collection: collection,
data: [],
});
}
}
}
async getAllUsers() {
const users = new UsersController(this.config, this.database);
const allUsers = await users.getAllUsers();
// Iterate over the users and setup our maps ahead of time for email and phone
for (const user of allUsers) {
if (user.email) {
this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
}
if (user.phone) {
this.phoneToUserIdMap.set(user.phone, user.$id);
}
this.userExistsMap.set(user.$id, true);
this.userIdSet.add(user.$id);
let importData = this.importMap.get(this.getCollectionKey("users"));
if (!importData) {
importData = {
data: [],
};
}
importData.data.push({
finalData: {
...user,
email: user.email?.toLowerCase(),
userId: user.$id,
docId: user.$id,
},
context: {
...user,
email: user.email?.toLowerCase(),
userId: user.$id,
docId: user.$id,
},
rawData: user,
});
this.importMap.set(this.getCollectionKey("users"), importData);
}
return allUsers;
}
// Main method to start the data loading process for a given database ID
async start(dbId) {
MessageFormatter.divider();
MessageFormatter.info(`Starting data setup for database: ${dbId}`, { prefix: "Data" });
MessageFormatter.divider();
await this.setupMaps(dbId);
const allUsers = await this.getAllUsers();
MessageFormatter.info(`Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`, { prefix: "Data" });
await new Promise((resolve) => setTimeout(resolve, 5000));
// Iterate over the configured databases to find the matching one
for (const db of this.config.databases) {
if (db.$id !== dbId) {
continue;
}
if (!this.config.collections) {
continue;
}
// Iterate over the configured collections to process each
for (const collectionConfig of this.config.collections) {
const collection = collectionConfig;
// Determine if this is the users collection
let isUsersCollection = this.getCollectionKey(this.config.usersCollectionName) ===
this.getCollectionKey(collection.name);
const collectionDefs = collection.importDefs;
if (!collectionDefs || !collectionDefs.length) {
continue;
}
// Process create and update definitions for the collection
const createDefs = collection.importDefs.filter((def) => def.type === "create" || !def.type);
const updateDefs = collection.importDefs.filter((def) => def.type === "update");
for (const createDef of createDefs) {
if (!isUsersCollection || !createDef.createUsers) {
await this.prepareCreateData(db, collection, createDef);
}
else {
// Special handling for users collection if needed
await this.prepareUserCollectionCreateData(db, collection, createDef);
}
}
for (const updateDef of updateDefs) {
if (!this.importMap.has(this.getCollectionKey(collection.name))) {
logger.error(`No data found for collection ${collection.name} for updateDef but it says it's supposed to have one...`);
continue;
}
// Prepare the update data for the collection
this.prepareUpdateData(db, collection, updateDef);
}
}
MessageFormatter.info("Running update references", { prefix: "Data" });
// this.dealWithMergedUsers();
this.updateOldReferencesForNew();
MessageFormatter.success("Done running update references", { prefix: "Data" });
}
// for (const collection of this.config.collections) {
// this.resolveDataItemRelationships(collection);
// }
MessageFormatter.divider();
MessageFormatter.success(`Data setup for database: ${dbId} completed`, { prefix: "Data" });
MessageFormatter.divider();
if (this.shouldWriteFile) {
this.writeMapsToJsonFile();
}
}
/**
* Deals with merged users by iterating through all collections in the configuration.
* We have merged users if there are duplicate emails or phones in the import data.
* This function will iterate through all collections that are the same name as the
* users collection and pull out their primaryKeyField's. It will then loop through
* each collection and find any documents that have a
*
* @return {void} This function does not return anything.
*/
// dealWithMergedUsers() {
// const usersCollectionKey = this.getCollectionKey(
// this.config.usersCollectionName
// );
// const usersCollectionData = this.importMap.get(usersCollectionKey);
// if (!this.config.collections) {
// console.log("No collections found in configuration.");
// return;
// }
// let needsUpdate = false;
// let numUpdates = 0;
// for (const collectionConfig of this.config.collections) {
// const collectionKey = this.getCollectionKey(collectionConfig.name);
// const collectionData = this.importMap.get(collectionKey);
// const collectionImportDefs = collectionConfig.importDefs;
// const collectionIdMappings = collectionImportDefs
// .map((importDef) => importDef.idMappings)
// .flat()
// .filter((idMapping) => idMapping !== undefined && idMapping !== null);
// if (!collectionData || !collectionData.data) continue;
// for (const dataItem of collectionData.data) {
// for (const idMapping of collectionIdMappings) {
// // We know it's the users collection here
// if (this.getCollectionKey(idMapping.targetCollection) === usersCollectionKey) {
// const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
// if (targetFieldKey === )
// const targetValue = dataItem.finalData[targetFieldKey];
// const targetCollectionData = this.importMap.get(this.getCollectionKey(idMapping.targetCollection));
// if (!targetCollectionData || !targetCollectionData.data) continue;
// const foundData = targetCollectionData.data.filter(({ context }) => {
// const targetValue = context[targetFieldKey];
// const isMatch = `${targetValue}` === `${valueToMatch}`;
// return isMatch && targetValue !== undefined && targetValue !== null;
// });
// }
// }
// }
// }
// }
/**
* Gets the value to match for a given key in the final data or context.
* @param finalData - The final data object.
* @param context - The context object.
* @param key - The key to get the value for.
* @returns The value to match for from finalData or Context
*/
getValueFromData(finalData, context, key) {
if (context[key] !== undefined &&
context[key] !== null &&
context[key] !== "") {
return context[key];
}
return finalData[key];
}
updateOldReferencesForNew() {
if (!this.config.collections) {
return;
}
for (const collectionConfig of this.config.collections) {
const collectionKey = this.getCollectionKey(collectionConfig.name);
const collectionData = this.importMap.get(collectionKey);
if (!collectionData || !collectionData.data)
continue;
MessageFormatter.processing(`Updating references for collection: ${collectionConfig.name}`, { prefix: "Data" });
let needsUpdate = false;
// Iterate over each data item in the current collection
for (let i = 0; i < collectionData.data.length; i++) {
if (collectionConfig.importDefs) {
for (const importDef of collectionConfig.importDefs) {
if (importDef.idMappings) {
for (const idMapping of importDef.idMappings) {
const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
const sourceValue = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
// Skip if value to match is missing or empty
if (!sourceValue ||
isEmpty(sourceValue) ||
sourceValue === null)
continue;
const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
const targetCollectionData = this.importMap.get(targetCollectionKey);
if (!targetCollectionData || !targetCollectionData.data)
continue;
// Handle cases where sourceValue is an array
const sourceValues = Array.isArray(sourceValue)
? sourceValue.map((sourceValue) => `${sourceValue}`)
: [`${sourceValue}`];
let newData = [];
for (const valueToMatch of sourceValues) {
// Find matching data in the target collection
const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
const isMatch = `${targetValue}` === `${valueToMatch}`;
// Ensure the targetValue is defined and not null
return (isMatch &&
targetValue !== undefined &&
targetValue !== null);
});
if (foundData.length) {
newData.push(...foundData.map((data) => {
const newValue = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
return newValue;
}));
}
else {
logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
}
continue;
}
const getCurrentDataFiltered = (currentData) => {
if (Array.isArray(currentData.finalData[fieldToSetKey])) {
return currentData.finalData[fieldToSetKey].filter((data) => !sourceValues.includes(`${data}`));
}
return currentData.finalData[fieldToSetKey];
};
// Get the current data to be updated
const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
if (newData.length) {
needsUpdate = true;
// Handle cases where current data is an array
if (isFieldToSetArray) {
if (!currentDataFiltered) {
// Set new data if current data is undefined
collectionData.data[i].finalData[fieldToSetKey] =
Array.isArray(newData) ? newData : [newData];
}
else {
if (Array.isArray(currentDataFiltered)) {
// Convert current data to array and merge if new data is non-empty array
collectionData.data[i].finalData[fieldToSetKey] = [
...new Set([...currentDataFiltered, ...newData].filter((value) => value !== null &&
value !== undefined &&
value !== "")),
];
}
else {
// Merge arrays if new data is non-empty array and filter for uniqueness
collectionData.data[i].finalData[fieldToSetKey] = [
...new Set([
...(Array.isArray(currentDataFiltered)
? currentDataFiltered
: [currentDataFiltered]),
...newData,
].filter((value) => value !== null &&
value !== undefined &&
value !== "" &&
!sourceValues.includes(`${value}`))),
];
}
}
}
else {
if (!currentDataFiltered) {
// Set new data if current data is undefined
collectionData.data[i].finalData[fieldToSetKey] =
Array.isArray(newData) ? newData[0] : newData;
}
else if (Array.isArray(newData) && newData.length > 0) {
// Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
// and take the first value, because it's an array and the attribute is not an array
collectionData.data[i].finalData[fieldToSetKey] = [
...new Set([currentDataFiltered, ...newData].filter((value) => value !== null &&
value !== undefined &&
value !== "" &&
!sourceValues.includes(`${value}`))),
].slice(0, 1)[0];
}
else if (!Array.isArray(newData) &&
newData !== undefined) {
// Simply update the field if new data is not an array and defined
collectionData.data[i].finalData[fieldToSetKey] = newData;
}
}
}
}
}
}
}
}
// Update the import map if any changes were made
if (needsUpdate) {
this.importMap.set(collectionKey, collectionData);
}
}
}
writeMapsToJsonFile() {
const outputDir = path.resolve(process.cwd(), "zlogs");
// Ensure the logs directory exists
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir);
}
// Helper function to write data to a file
const writeToFile = (fileName, data) => {
const outputFile = path.join(outputDir, fileName);
fs.writeFile(outputFile, JSON.stringify(data, null, 2), "utf8", (err) => {
if (err) {
MessageFormatter.error(`Error writing data to ${fileName}`, err instanceof Error ? err : new Error(String(err)), { prefix: "Data" });
return;
}
MessageFormatter.success(`Data successfully written to ${fileName}`, { prefix: "Data" });
});
};
// Convert Maps to arrays of entries for serialization
const oldIdToNewIdPerCollectionMap = Array.from(this.oldIdToNewIdPerCollectionMap.entries()).map(([key, value]) => {
return {
collection: key,
data: Array.from(value.entries()),
};
});
const mergedUserMap = Array.from(this.mergedUserMap.entries());
// Write each part to a separate file
writeToFile("oldIdToNewIdPerCollectionMap.json", oldIdToNewIdPerCollectionMap);
writeToFile("mergedUserMap.json", mergedUserMap);
// Write each collection's data to a separate file
this.importMap.forEach((value, key) => {
const data = {
collection: key,
data: value.data.map((item) => {
return {
finalData: item.finalData,
context: item.context,
};
}),
};
writeToFile(`${key}.json`, data);
});
}
/**
* Prepares user data by checking for duplicates based on email or phone, adding to a duplicate map if found,
* and then returning the transformed item without user-specific keys.
*
* @param item - The raw item to be processed.
* @param attributeMappings - The attribute mappings for the item.
* @returns The transformed item with user-specific keys removed.
*/
prepareUserData(item, attributeMappings, primaryKeyField, newId) {
if (this.userIdSet.has(newId) ||
this.userExistsMap.has(newId) ||
Array.from(this.emailToUserIdMap.values()).includes(newId) ||
Array.from(this.phoneToUserIdMap.values()).includes(newId)) {
newId = this.getTrueUniqueId(this.getCollectionKey("users"));
}
let transformedItem = this.transformData(item, attributeMappings);
let userData = AuthUserCreateSchema.safeParse(transformedItem);
if (userData.data?.email) {
userData.data.email = userData.data.email.toLowerCase();
}
if (!userData.success || !(userData.data.email || userData.data.phone)) {
logger.error(`Invalid user data: ${JSON.stringify(userData.error?.issues, undefined, 2)} or missing email/phone`);
const userKeys = ["email", "phone", "name", "labels", "prefs"];
userKeys.forEach((key) => {
if (transformedItem.hasOwnProperty(key)) {
delete transformedItem[key];
}
});
return {
transformedItem,
existingId: undefined,
userData: {
rawData: item,
finalData: transformedItem,
},
};
}
const email = userData.data.email?.toLowerCase();
const phone = userData.data.phone;
let existingId;
// Check for duplicate email and phone
if (email && this.emailToUserIdMap.has(email)) {
existingId = this.emailToUserIdMap.get(email);
if (phone && !this.phoneToUserIdMap.has(phone)) {
this.phoneToUserIdMap.set(phone, newId);
}
}
else if (phone && this.phoneToUserIdMap.has(phone)) {
existingId = this.phoneToUserIdMap.get(phone);
if (email && !this.emailToUserIdMap.has(email)) {
this.emailToUserIdMap.set(email, newId);
}
}
else {
if (email)
this.emailToUserIdMap.set(email, newId);
if (phone)
this.phoneToUserIdMap.set(phone, newId);
}
if (existingId) {
userData.data.userId = existingId;
const mergedUsers = this.mergedUserMap.get(existingId) || [];
mergedUsers.push(`${item[primaryKeyField]}`);
this.mergedUserMap.set(existingId, mergedUsers);
const userFound = this.importMap
.get(this.getCollectionKey("users"))
?.data.find((userDataExisting) => {
let userIdToMatch;
if (userDataExisting?.finalData?.userId) {
userIdToMatch = userDataExisting?.finalData?.userId;
}
else if (userDataExisting?.finalData?.docId) {
userIdToMatch = userDataExisting?.finalData?.docId;
}
else if (userDataExisting?.context?.userId) {
userIdToMatch = userDataExisting.context.userId;
}
else if (userDataExisting?.context?.docId) {
userIdToMatch = userDataExisting.context.docId;
}
return userIdToMatch === existingId;
});
if (userFound) {
userFound.finalData.userId = existingId;
userFound.finalData.docId = existingId;
this.userIdSet.add(existingId);
transformedItem = {
...transformedItem,
userId: existingId,
docId: existingId,
};
}
const userKeys = ["email", "phone", "name", "labels", "prefs"];
userKeys.forEach((key) => {
if (transformedItem.hasOwnProperty(key)) {
delete transformedItem[key];
}
});
return {
transformedItem,
existingId,
userData: {
rawData: userFound?.rawData,
finalData: userFound?.finalData,
},
};
}
else {
existingId = newId;
userData.data.userId = existingId;
}
const userKeys = ["email", "phone", "name", "labels", "prefs"];
userKeys.forEach((key) => {
if (transformedItem.hasOwnProperty(key)) {
delete transformedItem[key];
}
});
const usersMap = this.importMap.get(this.getCollectionKey("users"));
const userDataToAdd = {
rawData: item,
finalData: userData.data,
context: {},
};
this.importMap.set(this.getCollectionKey("users"), {
data: [...(usersMap?.data || []), userDataToAdd],
});
this.userIdSet.add(existingId);
return {
transformedItem,
existingId,
userData: userDataToAdd,
};
}
/**
* Prepares the data for creating user collection documents.
* This involves loading the data, transforming it according to the import definition,
* and handling the creation of new unique IDs for each item.
*
* @param db - The database configuration.
* @param collection - The collection configuration.
* @param importDef - The import definition containing the attribute mappings and other relevant info.
*/
async prepareUserCollectionCreateData(db, collection, importDef) {
// Load the raw data based on the import definition
const rawData = this.loadData(importDef);
let operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
// Initialize a new map for old ID to new ID mappings
const oldIdToNewIdMap = new Map();
// Retrieve or initialize the collection-specific old ID to new ID map
const collectionOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name)) ||
this.oldIdToNewIdPerCollectionMap
.set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
.get(this.getCollectionKey(collection.name));
const adapter = new LegacyAdapter(this.database.client);
if (!operationId) {
const collectionImportOperation = await findOrCreateOperation(adapter, db.$id, "importData", collection.$id);
// Store the operation ID in the map
this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
operationId = collectionImportOperation.$id;
}
if (operationId) {
await updateOperation(adapter, db.$id, operationId, {
status: "ready",
total: rawData.length,
});
}
// Retrieve the current user data and the current collection data from the import map
const currentUserData = this.importMap.get(this.getCollectionKey("users"));
const currentData = this.importMap.get(this.getCollectionKey(collection.name));
// Log errors if the necessary data is not found in the import map
if (!currentUserData) {
logger.error(`No data found for collection ${"users"} for createDef but it says it's supposed to have one...`);
return;
}
else if (!currentData) {
logger.error(`No data found for collection ${collection.name} for createDef but it says it's supposed to have one...`);
return;
}
// Iterate through each item in the raw data
for (const item of rawData) {
// Prepare user data, check for duplicates, and remove user-specific keys
let { transformedItem, existingId, userData } = this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
logger.info(`In create user -- transformedItem: ${JSON.stringify(transformedItem, null, 2)}`);
// Generate a new unique ID for the item or use existing ID
if (!existingId && !userData.finalData?.userId) {
// No existing user ID, generate a new unique ID
existingId = this.getTrueUniqueId(this.getCollectionKey(collection.name));
transformedItem = {
...transformedItem,
userId: existingId,
docId: existingId,
};
}
else if (!existingId && userData.finalData?.userId) {
// Existing user ID, use it as the new ID
existingId = userData.finalData.userId;
transformedItem = {
...transformedItem,
userId: existingId,
docId: existingId,
};
}
// Create a context object for the item, including the new ID
let context = this.createContext(db, collection, item, existingId);
// Merge the transformed data into the context
context = { ...context, ...transformedItem, ...userData.finalData };
// If a primary key field is defined, handle the ID mapping and check for duplicates
if (importDef.primaryKeyField) {
const oldId = item[importDef.primaryKeyField];
// Check if the oldId already exists to handle potential duplicates
if (this.oldIdToNewIdPerCollectionMap
.get(this.getCollectionKey(collection.name))
?.has(`${oldId}`)) {
// Found a duplicate oldId, now decide how to merge or handle these duplicates
for (const data of currentData.data) {
if (data.finalData.docId === oldId ||
data.finalData.userId === oldId ||
data.context.docId === oldId ||
data.context.userId === oldId) {
transformedItem = this.mergeObjects(data.finalData, transformedItem);
}
}
}
else {
// No duplicate found, simply map the oldId to the new itemId
collectionOldIdToNewIdMap?.set(`${oldId}`, `${existingId}`);
}
}
// Handle merging for currentUserData
for (let i = 0; i < currentUserData.data.length; i++) {
const currentUserDataItem = currentUserData.data[i];
const samePhones = currentUserDataItem.finalData.phone &&
transformedItem.phone &&
currentUserDataItem.finalData.phone === transformedItem.phone;
const sameEmails = currentUserDataItem.finalData.email &&
transformedItem.email &&
currentUserDataItem.finalData.email === transformedItem.email;
if ((currentUserDataItem.finalData.docId === existingId ||
currentUserDataItem.finalData.userId === existingId) &&
(samePhones || sameEmails) &&
currentUserDataItem.finalData &&
userData.finalData) {
const userDataMerged = this.mergeObjects(currentUserData.data[i].finalData, userData.finalData);
currentUserData.data[i].finalData = userDataMerged;
this.importMap.set(this.getCollectionKey("users"), currentUserData);
}
}
// Update the attribute mappings with any actions that need to be performed post-import
// We added the basePath to get the folder from the filePath
const mappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, context, transformedItem);
// Update the import definition with the new attribute mappings
const newImportDef = {
...importDef,
attributeMappings: mappingsWithActions,
};
const updatedData = this.importMap.get(this.getCollectionKey(collection.name));
let foundData = false;
for (let i = 0; i < updatedData.data.length; i++) {
if (updatedData.data[i].finalData.docId === existingId ||
updatedData.data[i].finalData.userId === existingId ||
updatedData.data[i].context.docId === existingId ||
updatedData.data[i].context.userId === existingId) {
updatedData.data[i].finalData = this.mergeObjects(updatedData.data[i].finalData, transformedItem);
updatedData.data[i].context = this.mergeObjects(updatedData.data[i].context, context);
const mergedImportDef = {
...updatedData.data[i].importDef,
idMappings: [
...(updatedData.data[i].importDef?.idMappings || []),
...(newImportDef.idMappings || []),
],
attributeMappings: [
...(updatedData.data[i].importDef?.attributeMappings || []),
...(newImportDef.attributeMappings || []),
],
};
updatedData.data[i].importDef = mergedImportDef;
this.importMap.set(this.getCollectionKey(collection.name), updatedData);
this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
foundData = true;
}
}
if (!foundData) {
// Add new data to the associated collection
updatedData.data.push({
rawData: item,
context: context,
importDef: newImportDef,
finalData: transformedItem,
});
this.importMap.set(this.getCollectionKey(collection.name), updatedData);
this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
}
}
}
/**
* Prepares the data for creating documents in a collection.
* This involves loading the data, transforming it, and handling ID mappings.
*
* @param db - The database configuration.
* @param collection - The collection configuration.
* @param importDef - The import definition containing the attribute mappings and other relevant info.
*/
async prepareCreateData(db, collection, importDef) {
// Load the raw data based on the import definition
const rawData = this.loadData(importDef);
let operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
const adapter = new LegacyAdapter(this.database.client);
if (!operationId) {
const collectionImportOperation = await findOrCreateOperation(adapter, db.$id, "importData", collection.$id);
// Store the operation ID in the map
this.collectionImportOperations.set(this.getCollectionKey(collection.name), collectionImportOperation.$id);
operationId = collectionImportOperation.$id;
}
if (operationId) {
await updateOperation(adapter, db.$id, operationId, {
status: "ready",
total: rawData.length,
});
}
// Initialize a new map for old ID to new ID mappings
const oldIdToNewIdMapNew = new Map();
// Retrieve or initialize the collection-specific old ID to new ID map
const collectionOldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name)) ||
this.oldIdToNewIdPerCollectionMap
.set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
.get(this.getCollectionKey(collection.name));
const isRegions = collection.name.toLowerCase() === "regions";
// Iterate through each item in the raw data
for (const item of rawData) {
// Generate a new unique ID for the item
const itemIdNew = this.getTrueUniqueId(this.getCollectionKey(collection.name));
if (isRegions) {
logger.info(`Creating region: ${JSON.stringify(item, null, 2)}`);
}
// Retrieve the current collection data from the import map
const currentData = this.importMap.get(this.getCollectionKey(collection.name));
// Create a context object for the item, including the new ID
let context = this.createContext(db, collection, item, itemIdNew);
// Transform the item data based on the attribute mappings
let transformedData = this.transformData(item, importDef.attributeMappings);
// If a primary key field is defined, handle the ID mapping and check for duplicates
if (importD