@directus/api/dist/services/import/import.js

Directus is a real-time API and App dashboard for managing SQL database content

import { useLogger } from "../../logger/index.js";
import { getCache } from "../../cache.js";
import database_default from "../../database/index.js";
import emitter_default from "../../emitter.js";
import { validateAccess } from "../../permissions/modules/validate-access/validate-access.js";
import { transaction } from "../../utils/transaction.js";
import { createMutationTracker } from "../../utils/create-mutation-tracker.js";
import { shouldClearCache } from "../../utils/should-clear-cache.js";
import { getService } from "../../utils/get-service.js";
import { userName } from "../../utils/user-name.js";
import { UsersService } from "../users.js";
import { NotificationsService } from "../notifications.js";
import { setDeep } from "../../utils/set-deep.js";
import { useStore } from "../../utils/store.js";
import { buildImportPlan } from "../../utils/build-import-plan.js";
import { destroyPipedStream } from "../../utils/destroy-piped-stream.js";
import { createErrorTracker } from "../../utils/error-tracker.js";
import { keyExists } from "./key-exists.js";
import { normalizeKey } from "./normalize-key.js";
import { remapForeignKeys, remapValue, resolveTarget } from "./remap-foreign-keys.js";
import { validateFlatData } from "./validate-flat-data.js";
import { useEnv } from "@directus/env";
import { ContentTooLargeError, ForbiddenError, InvalidPayloadError, InvalidQueryError, LimitExceededError, TimeoutError, UnsupportedMediaTypeError } from "@directus/errors";
import { parseJSON, toArray } from "@directus/utils";
import { createTmpFile } from "@directus/utils/node";
import ms from "ms";
import { randomUUID } from "node:crypto";
import { isSystemCollection } from "@directus/system-data";
import { pipeline } from "node:stream/promises";
import { createReadStream, createWriteStream } from "node:fs";
import { queue } from "async";
import Papa from "papaparse";
import StreamArray from "stream-json/streamers/StreamArray.js";

//#region src/services/import/import.ts
const env = useEnv();
const logger = useLogger();
const store = useStore(String(env["IMPORT_EXPORT_NAMESPACE"]), { ttl: ms(env["IMPORT_TIMEOUT"] ?? "1h") });
var DryRunRollback = class extends Error {};
var ImportService = class {
	knex;
	accountability;
	schema;
	constructor(options) {
		this.knex = options.knex || database_default();
		this.accountability = options.accountability || null;
		this.schema = options.schema;
	}
	async acquireImportSlot() {
		if (await store(async (store$1) => {
			const count = await store$1.get("importCount") ?? 0;
			if (count >= Number(env["IMPORT_MAX_CONCURRENCY"])) return true;
			await store$1.set("importCount", count + 1);
			return false;
		})) throw new LimitExceededError({ category: "Concurrent import" });
	}
	async releaseImportSlot() {
		try {
			await store(async (store$1) => {
				const count = await store$1.get("importCount") ?? 0;
				await store$1.set("importCount", count - 1);
			});
		} catch (error) {
			logger.error(error, `Failed to decrement importCount`);
		}
	}
	async import(collection, mimetype, stream, options) {
		if (this.accountability?.admin !== true && isSystemCollection(collection)) throw new ForbiddenError();
		if (this.accountability) {
			await validateAccess({
				accountability: this.accountability,
				action: "create",
				collection
			}, {
				schema: this.schema,
				knex: this.knex
			});
			await validateAccess({
				accountability: this.accountability,
				action: "update",
				collection
			}, {
				schema: this.schema,
				knex: this.knex
			});
		}
		if ([
			"application/json",
			"text/csv",
			"application/vnd.ms-excel"
		].includes(mimetype) === false) throw new UnsupportedMediaTypeError({
			mediaType: mimetype,
			where: "file import"
		});
		await this.acquireImportSlot();
		let promise;
		if (options?.background) {
			let tmpFile;
			const deadline = Date.now() + ms(env["IMPORT_TIMEOUT"]);
			try {
				tmpFile = await this.spoolToTmpFile(stream, deadline);
			} catch (error) {
				await this.releaseImportSlot();
				throw error;
			}
			if (mimetype === "application/json") promise = this.importJSON(collection, createReadStream(tmpFile.path), deadline).finally(() => tmpFile.cleanup().catch(() => {
				logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
			}));
			else promise = this.parseCsvFromTmpFile(collection, tmpFile, deadline);
		} else if (mimetype === "application/json") promise = this.importJSON(collection, stream);
		else promise = this.importCSV(collection, stream);
		if (options?.background) {
			const notify = async (subject, message) => {
				try {
					if (!this.accountability?.user) return;
					const notificationsService = new NotificationsService({ schema: this.schema });
					const user = await new UsersService({ schema: this.schema }).readOne(this.accountability.user, { fields: [
						"first_name",
						"last_name",
						"email"
					] });
					await notificationsService.createOne({
						recipient: this.accountability.user,
						sender: this.accountability.user,
						subject,
						message: `Hello ${userName(user)},\n\n${message}\n`
					});
				} catch (error) {
					logger.error(error, `Failed to notify user`);
				}
			};
			promise.then(async () => {
				await notify("Your import has been successful", `Your import in ${collection} has been successful.`);
			}).catch(async (error) => {
				logger.error(error, `Background import to ${collection} failed`);
				await notify("Your import has failed", `Your import in ${collection} has failed.\n\n${error.message ?? ""}`);
			}).finally(async () => await this.releaseImportSlot());
		} else try {
			await promise;
		} finally {
			await this.releaseImportSlot();
		}
	}
	async importJSON(collection, stream, deadline) {
		const extractJSON = StreamArray.withParser();
		const nestedActionEvents = [];
		const errorTracker = createErrorTracker();
		const isSingleton = this.schema.collections[collection]?.singleton ?? false;
		let timeout;
		const teardownStreams = () => {
			destroyPipedStream(extractJSON, stream);
		};
		return transaction(this.knex, async (trx) => {
			const service = getService(collection, {
				knex: trx,
				schema: this.schema,
				accountability: this.accountability
			});
			try {
				await new Promise((resolve, reject) => {
					let rowNumber = 1;
					const saveQueue = queue(async (task) => {
						if (errorTracker.shouldStop()) return;
						try {
							if (isSingleton) return await service.upsertSingleton(task.data, { bypassEmitAction: (params) => nestedActionEvents.push(params) });
							else return await service.upsertOne(task.data, { bypassEmitAction: (params) => nestedActionEvents.push(params) });
						} catch (error) {
							for (const err of toArray(error)) {
								errorTracker.addCapturedError(err, task.rowNumber);
								if (errorTracker.shouldStop()) break;
							}
							if (errorTracker.shouldStop()) {
								saveQueue.kill();
								teardownStreams();
								reject();
							}
							return;
						}
					});
					stream.pipe(extractJSON);
					stream.on("error", (error) => {
						saveQueue.kill();
						teardownStreams();
						reject(error instanceof ContentTooLargeError ? error : new Error("Error while retrieving import data", { cause: error }));
					});
					extractJSON.on("data", ({ value }) => {
						if (isSingleton && rowNumber > 1) {
							saveQueue.kill();
							teardownStreams();
							reject(new InvalidPayloadError({ reason: `Cannot import multiple records into singleton collection ${collection}` }));
							return;
						}
						saveQueue.push({
							data: value,
							rowNumber: rowNumber++
						});
					});
					extractJSON.on("error", (err) => {
						teardownStreams();
						reject(new InvalidPayloadError({ reason: err.message }));
					});
					extractJSON.on("end", () => {
						if (!saveQueue.started) return resolve();
						saveQueue.drain(() => {
							if (errorTracker.hasErrors()) return reject();
							for (const nestedActionEvent of nestedActionEvents) emitter_default.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
							return resolve();
						});
					});
					const duration = ms(env["IMPORT_TIMEOUT"]);
					const delay = deadline !== void 0 ? Math.max(0, deadline - Date.now()) : duration;
					timeout = setTimeout(() => {
						saveQueue.kill();
						teardownStreams();
						reject(new TimeoutError({
							category: "Import",
							duration
						}));
					}, delay);
				});
			} catch (error) {
				if (!error && errorTracker.hasErrors()) throw errorTracker.buildFinalErrors();
				throw error;
			} finally {
				clearTimeout(timeout);
			}
		});
	}
	/**
	* Spool a source stream to a fresh temp file, resolving only once the whole stream has been
	* received. Used to fully consume the request body within the request lifecycle before a
	* background import detaches (see the background branch in `import()`).
	*/
	async spoolToTmpFile(stream, deadline) {
		const tmpFile = await createTmpFile().catch(() => null);
		if (!tmpFile) throw new Error("Failed to create temporary file for import");
		const duration = ms(env["IMPORT_TIMEOUT"]);
		const delay = deadline !== void 0 ? Math.max(0, deadline - Date.now()) : duration;
		const controller = new AbortController();
		const timeout = setTimeout(() => controller.abort(), delay);
		try {
			await pipeline(stream, createWriteStream(tmpFile.path), { signal: controller.signal });
		} catch (error) {
			await tmpFile.cleanup().catch(() => {
				logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
			});
			if (error instanceof ContentTooLargeError) throw error;
			if (controller.signal.aborted) throw new TimeoutError({
				category: "Import",
				duration
			});
			throw new Error("Error while retrieving import data", { cause: error });
		} finally {
			clearTimeout(timeout);
		}
		return tmpFile;
	}
	async importCSV(collection, stream) {
		const deadline = Date.now() + ms(env["IMPORT_TIMEOUT"]);
		const tmpFile = await this.spoolToTmpFile(stream, deadline);
		return this.parseCsvFromTmpFile(collection, tmpFile, deadline);
	}
	/**
	* Parse an already-spooled CSV temp file and upsert its rows. Owns the lifecycle of the passed
	* temp file and cleans it up when done.
	*/
	async parseCsvFromTmpFile(collection, tmpFile, deadline) {
		const nestedActionEvents = [];
		const errorTracker = createErrorTracker();
		const isSingleton = this.schema.collections[collection]?.singleton ?? false;
		let timeout;
		let removed = false;
		const removeTmpFile = () => {
			if (removed) return;
			removed = true;
			tmpFile.cleanup().catch(() => {
				logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
			});
		};
		return transaction(this.knex, async (trx) => {
			const service = getService(collection, {
				knex: trx,
				schema: this.schema,
				accountability: this.accountability
			});
			try {
				await new Promise((resolve, reject) => {
					const streams = [];
					let rowNumber = 0;
					const cleanup = () => {
						for (const stream of streams) stream.destroy();
						removeTmpFile();
					};
					const saveQueue = queue(async (task) => {
						if (errorTracker.shouldStop()) return;
						try {
							if (isSingleton) return await service.upsertSingleton(task.data, { bypassEmitAction: (action) => nestedActionEvents.push(action) });
							else return await service.upsertOne(task.data, { bypassEmitAction: (action) => nestedActionEvents.push(action) });
						} catch (error) {
							for (const err of toArray(error)) {
								errorTracker.addCapturedError(err, task.rowNumber);
								if (errorTracker.shouldStop()) break;
							}
							if (errorTracker.shouldStop()) {
								saveQueue.kill();
								cleanup();
								reject();
							}
							return;
						}
					});
					const fileReadStream = createReadStream(tmpFile.path).on("error", (error) => {
						cleanup();
						reject(new Error("Error while reading import data from temporary file", { cause: error }));
					});
					streams.push(fileReadStream);
					const parseStream = Papa.parse(Papa.NODE_STREAM_INPUT, {
						header: true,
						transformHeader: (header) => header.trim(),
						transform: (value) => {
							if (value.length === 0) return;
							try {
								const parsedJson = parseJSON(value);
								if (typeof parsedJson === "number") return value;
								return parsedJson;
							} catch {
								return value;
							}
						}
					});
					fileReadStream.pipe(parseStream).on("data", (obj) => {
						rowNumber++;
						if (isSingleton && rowNumber > 1) {
							saveQueue.kill();
							cleanup();
							reject(new InvalidPayloadError({ reason: `Cannot import multiple records into singleton collection ${collection}` }));
							return;
						}
						const result = Object.create(null);
						for (const field in obj) if (obj[field] !== void 0) setDeep(result, field, obj[field]);
						saveQueue.push({
							data: result,
							rowNumber
						});
					}).on("error", (error) => {
						cleanup();
						reject(new InvalidPayloadError({ reason: error.message }));
					}).on("end", () => {
						if (!saveQueue.started) {
							removeTmpFile();
							return resolve();
						}
						saveQueue.drain(() => {
							if (!errorTracker.shouldStop()) removeTmpFile();
							if (errorTracker.hasErrors()) return reject();
							for (const nestedActionEvent of nestedActionEvents) emitter_default.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
							return resolve();
						});
					});
					const duration = ms(env["IMPORT_TIMEOUT"]);
					const delay = deadline !== void 0 ? Math.max(0, deadline - Date.now()) : duration;
					timeout = setTimeout(() => {
						saveQueue.kill();
						destroyPipedStream(parseStream, fileReadStream);
						cleanup();
						reject(new TimeoutError({
							category: "Import",
							duration
						}));
					}, delay);
				});
			} catch (error) {
				if (!error && errorTracker.hasErrors()) throw errorTracker.buildFinalErrors();
				throw error;
			} finally {
				clearTimeout(timeout);
			}
		}).finally(() => removeTmpFile());
	}
	/**
	* Import data for multiple collections in a single request. Builds a relational dependency graph
	* from the schema, imports collections in topological order, remaps primary keys (and the foreign
	* keys that reference them), and resolves nullable relational cycles via a second pass.
	*/
	async importBatch(input, options = {}) {
		const mode = options.mode ?? "add";
		const dryRun = options.dryRun ?? false;
		const dangerouslyAllowDelete = options.dangerouslyAllowDelete ?? false;
		if (dangerouslyAllowDelete && mode !== "merge") throw new InvalidQueryError({ reason: `"dangerouslyAllowDelete" can only be used with mode "merge"` });
		const plan = buildImportPlan(input, this.schema);
		const dataByCollection = /* @__PURE__ */ new Map();
		for (const entry of input) dataByCollection.set(entry.collection, entry);
		const collections = {};
		const idMaps = /* @__PURE__ */ new Map();
		const newPksByCollection = /* @__PURE__ */ new Map();
		for (const collection of plan.order) {
			collections[collection] = {
				existing: [],
				new: [],
				deleted: [],
				mapped: {}
			};
			idMaps.set(collection, /* @__PURE__ */ new Map());
			newPksByCollection.set(collection, []);
			if (this.accountability?.admin !== true && isSystemCollection(collection)) throw new ForbiddenError();
			if (this.accountability) {
				await validateAccess({
					accountability: this.accountability,
					action: "create",
					collection
				}, {
					schema: this.schema,
					knex: this.knex
				});
				const hasRemappableAlias = plan.aliasFields.get(collection).some((info) => info.target !== null);
				const isSingleton = this.schema.collections[collection]?.singleton ?? false;
				if (mode === "merge" || isSingleton || plan.deferred.has(collection) || hasRemappableAlias) await validateAccess({
					accountability: this.accountability,
					action: "update",
					collection
				}, {
					schema: this.schema,
					knex: this.knex
				});
				if (dangerouslyAllowDelete) await validateAccess({
					accountability: this.accountability,
					action: "delete",
					collection
				}, {
					schema: this.schema,
					knex: this.knex
				});
			}
		}
		validateFlatData(plan.fkFields, plan.aliasFields, dataByCollection);
		const nestedActionEvents = [];
		await this.acquireImportSlot();
		try {
			await transaction(this.knex, async (trx) => {
				const mutationOptions = {
					bypassEmitAction: (params) => nestedActionEvents.push(params),
					mutationTracker: createMutationTracker(),
					autoPurgeCache: false,
					autoPurgeSystemCache: false
				};
				for (const collection of plan.order) {
					const entry = dataByCollection.get(collection);
					const idMap = idMaps.get(collection);
					const newPks = newPksByCollection.get(collection);
					const fkFields = plan.fkFields.get(collection);
					const secondPassFields = new Set(plan.deferred.get(collection));
					for (const info of plan.aliasFields.get(collection) ?? []) if (info.target !== null) secondPassFields.add(info.field);
					const service = getService(collection, {
						knex: trx,
						schema: this.schema,
						accountability: this.accountability
					});
					const { primary: pkField, fields, singleton: isSingleton } = this.schema.collections[collection];
					const pkOverview = fields[pkField];
					const isAutoIncrement = ["integer", "bigInteger"].includes(pkOverview.type) && pkOverview.defaultValue === "AUTO_INCREMENT";
					const isUuid = pkOverview.type === "uuid";
					if (isSingleton && entry.items.length > 1) throw new InvalidPayloadError({ reason: `Cannot import multiple records into singleton collection "${collection}"` });
					for (const item of entry.items) {
						const payload = remapForeignKeys(item, fkFields, idMaps, secondPassFields);
						const oldPk = item[pkField];
						let newPk;
						let matchedExisting = false;
						if (isSingleton) {
							delete payload[pkField];
							const existingRow = await trx.select(pkField).from(collection).first();
							if (existingRow) {
								matchedExisting = true;
								newPk = await service.updateOne(existingRow[pkField], payload, mutationOptions);
							} else newPk = await service.createOne(payload, mutationOptions);
						} else if (mode === "merge") {
							matchedExisting = oldPk != null && await keyExists(trx, collection, pkField, oldPk);
							if (!matchedExisting && isAutoIncrement) delete payload[pkField];
							newPk = await service.upsertOne(payload, mutationOptions);
						} else if (isAutoIncrement) {
							delete payload[pkField];
							newPk = await service.createOne(payload, mutationOptions);
						} else {
							if (oldPk != null && await keyExists(trx, collection, pkField, oldPk)) if (isUuid) payload[pkField] = randomUUID();
							else throw new InvalidPayloadError({ reason: `Item with primary key "${oldPk}" in "${collection}" conflicts with an existing record and can't be safely remapped (only uuid keys are regenerated)` });
							newPk = await service.createOne(payload, mutationOptions);
						}
						newPks.push(newPk);
						const result = collections[collection];
						if (matchedExisting) result.existing.push(newPk);
						else result.new.push(newPk);
						if (oldPk != null) {
							idMap.set(String(oldPk), newPk);
							if (!isSingleton && normalizeKey(oldPk) !== normalizeKey(newPk)) result.mapped[String(oldPk)] = newPk;
						}
					}
				}
				for (const collection of plan.order) {
					const deferredFields = plan.deferred.get(collection);
					const aliasFields = plan.aliasFields.get(collection)?.filter((info) => info.target !== null) ?? [];
					if (!deferredFields && aliasFields.length === 0) continue;
					const entry = dataByCollection.get(collection);
					const newPks = newPksByCollection.get(collection);
					const fkFields = plan.fkFields.get(collection);
					const service = getService(collection, {
						knex: trx,
						schema: this.schema,
						accountability: this.accountability
					});
					for (let index = 0; index < entry.items.length; index++) {
						const item = entry.items[index];
						const newPk = newPks[index];
						if (newPk === void 0) continue;
						const patch = {};
						if (deferredFields) for (const field of deferredFields) {
							const rawValue = item[field];
							if (rawValue === void 0 || rawValue === null) continue;
							patch[field] = remapValue(rawValue, resolveTarget(fkFields.find((info) => info.field === field), item), idMaps);
						}
						for (const info of aliasFields) {
							const rawValue = item[info.field];
							if (rawValue === void 0 || rawValue === null) continue;
							patch[info.field] = remapValue(rawValue, info.target, idMaps);
						}
						if (Object.keys(patch).length === 0) continue;
						await service.updateOne(newPk, patch, mutationOptions);
					}
				}
				if (dangerouslyAllowDelete) for (const collection of [...plan.order].reverse()) {
					const result = collections[collection];
					const importKeys = [...result.existing, ...result.new];
					const { primary: pkField } = this.schema.collections[collection];
					result.deleted = await getService(collection, {
						knex: trx,
						schema: this.schema,
						accountability: this.accountability
					}).deleteByQuery({
						filter: importKeys.length > 0 ? { [pkField]: { _nin: importKeys } } : {},
						limit: -1
					}, mutationOptions);
				}
				if (dryRun) throw new DryRunRollback();
			});
			for (const nestedActionEvent of nestedActionEvents) emitter_default.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
			const { cache } = getCache();
			if (shouldClearCache(cache)) await cache.clear();
			return {
				applied: true,
				mode,
				collections
			};
		} catch (error) {
			if (error instanceof DryRunRollback) return {
				applied: false,
				mode,
				collections
			};
			throw error;
		} finally {
			await this.releaseImportSlot();
		}
	}
};

//#endregion
export { ImportService };