@directus/api/dist/services/export.js

Directus is a real-time API and App dashboard for managing SQL database content

import { useLogger } from "../logger/index.js";
import database_default from "../database/index.js";
import { transaction } from "../utils/transaction.js";
import { parseFields } from "../database/get-ast-from-query/lib/parse-fields.js";
import { FilesService } from "./files.js";
import { getService } from "../utils/get-service.js";
import { Url } from "../utils/url.js";
import { userName } from "../utils/user-name.js";
import { UsersService } from "./users.js";
import { NotificationsService } from "./notifications.js";
import { appendFile } from "node:fs/promises";
import { useEnv } from "@directus/env";
import { ServiceUnavailableError } from "@directus/errors";
import { getDateTimeFormatted } from "@directus/utils";
import { createReadStream } from "node:fs";
import { dump } from "js-yaml";
import { parse } from "js2xmlparser";
import { Parser, transforms } from "json2csv";

//#region src/services/export.ts
const env = useEnv();
const logger = useLogger();
var ExportService = class {
	knex;
	accountability;
	schema;
	constructor(options) {
		this.knex = options.knex || database_default();
		this.accountability = options.accountability || null;
		this.schema = options.schema;
	}
	/**
	* Export the query results as a named file. Will query in batches, and keep appending a tmp file
	* until all the data is retrieved. Uploads the result as a new file using the regular
	* FilesService upload method.
	*/
	async exportToFile(collection, query, format, options) {
		const { createTmpFile } = await import("@directus/utils/node");
		const tmpFile = await createTmpFile().catch(() => null);
		try {
			if (!tmpFile) throw new Error("Failed to create temporary file for export");
			const mimeTypes = {
				csv: "text/csv",
				csv_utf8: "text/csv; charset=utf-8",
				json: "application/json",
				xml: "text/xml",
				yaml: "text/yaml"
			};
			const database = database_default();
			await transaction(database, async (trx) => {
				const service = getService(collection, {
					accountability: this.accountability,
					schema: this.schema,
					knex: trx
				});
				const { primary } = this.schema.collections[collection];
				const sort = query.sort ?? [];
				if (sort.includes(primary) === false) sort.push(primary);
				const totalCount = await service.readByQuery({
					...query,
					aggregate: { count: ["*"] }
				}).then((result) => Number(result?.[0]?.["count"] ?? 0));
				const count = query.limit && query.limit > -1 ? Math.min(totalCount, query.limit) : totalCount;
				const requestedLimit = query.limit ?? -1;
				const batchesRequired = Math.ceil(count / env["EXPORT_BATCH_SIZE"]);
				let readCount = 0;
				for (let batch = 0; batch < batchesRequired; batch++) {
					let limit = env["EXPORT_BATCH_SIZE"];
					if (requestedLimit > 0 && env["EXPORT_BATCH_SIZE"] > requestedLimit - readCount) limit = requestedLimit - readCount;
					const result = await service.readByQuery({
						...query,
						sort,
						limit,
						offset: batch * env["EXPORT_BATCH_SIZE"]
					});
					readCount += result.length;
					if (result.length) {
						let csvHeadings = null;
						if (format.startsWith("csv")) {
							if (!query.fields) query.fields = ["*"];
							csvHeadings = getHeadingsForCsvExport(await parseFields({
								parentCollection: collection,
								fields: query.fields,
								query,
								accountability: this.accountability
							}, {
								schema: this.schema,
								knex: database
							}));
						}
						await appendFile(tmpFile.path, this.transform(result, format, {
							includeHeader: batch === 0,
							includeFooter: batch + 1 === batchesRequired,
							fields: csvHeadings
						}));
					}
				}
			});
			const filesService = new FilesService({
				accountability: this.accountability,
				schema: this.schema
			});
			const title = `export-${collection}-${getDateTimeFormatted()}`;
			const filename = `${title}.${format}`;
			const fileWithDefaults = {
				...options?.file ?? {},
				title: options?.file?.title ?? title,
				filename_download: options?.file?.filename_download ?? filename,
				type: mimeTypes[format]
			};
			const savedFile = await filesService.uploadOne(createReadStream(tmpFile.path), fileWithDefaults);
			if (this.accountability?.user) {
				const notificationsService = new NotificationsService({ schema: this.schema });
				const user = await new UsersService({ schema: this.schema }).readOne(this.accountability.user, { fields: [
					"first_name",
					"last_name",
					"email"
				] });
				const href = new Url(env["PUBLIC_URL"]).addPath("admin", "files", savedFile).toString();
				const message = `
Hello ${userName(user)},

Your export of ${collection} is ready. <a href="${href}">Click here to view.</a>
`;
				await notificationsService.createOne({
					recipient: this.accountability.user,
					sender: this.accountability.user,
					subject: `Your export of ${collection} is ready`,
					message,
					collection: `directus_files`,
					item: savedFile
				});
			}
		} catch (err) {
			logger.error(err, `Couldn't export ${collection}: ${err.message}`);
			if (this.accountability?.user) await new NotificationsService({ schema: this.schema }).createOne({
				recipient: this.accountability.user,
				sender: this.accountability.user,
				subject: `Your export of ${collection} failed`,
				message: `Please contact your system administrator for more information.`
			});
		} finally {
			await tmpFile?.cleanup();
		}
	}
	/**
	* Transform a given input object / array to the given type
	*/
	transform(input, format, options) {
		if (format === "json") {
			let string = JSON.stringify(input || null, null, "	");
			if (options?.includeHeader === false) string = string.split("\n").slice(1).join("\n");
			if (options?.includeFooter === false) {
				const lines = string.split("\n");
				string = lines.slice(0, lines.length - 1).join("\n");
				string += ",\n";
			}
			return string;
		}
		if (format === "xml") {
			let string = parse("data", input);
			if (options?.includeHeader === false) string = string.split("\n").slice(2).join("\n");
			if (options?.includeFooter === false) {
				const lines = string.split("\n");
				string = lines.slice(0, lines.length - 1).join("\n");
				string += "\n";
			}
			return string;
		}
		if (format.startsWith("csv")) {
			if (input.length === 0) return "";
			const transforms$1 = [transforms.flatten({ separator: "." })];
			const header = options?.includeHeader !== false;
			const withBOM = format === "csv_utf8";
			let string = new Parser(options?.fields ? {
				transforms: transforms$1,
				header,
				fields: options?.fields,
				withBOM
			} : {
				transforms: transforms$1,
				header,
				withBOM
			}).parse(input);
			if (options?.includeHeader === false) string = "\n" + string;
			return string;
		}
		if (format === "yaml") return dump(input);
		throw new ServiceUnavailableError({
			service: "export",
			reason: `Illegal export type used: "${format}"`
		});
	}
};
function getHeadingsForCsvExport(nodes, prefix = "") {
	let fieldNames = [];
	if (!nodes) return fieldNames;
	nodes.forEach((node) => {
		switch (node.type) {
			case "field":
			case "functionField":
			case "o2m":
			case "a2o":
				fieldNames.push(prefix ? `${prefix}.${node.fieldKey}` : node.fieldKey);
				break;
			case "m2o": fieldNames = fieldNames.concat(getHeadingsForCsvExport(node.children, prefix ? `${prefix}.${node.fieldKey}` : node.fieldKey));
		}
	});
	return fieldNames;
}

//#endregion
export { ExportService, getHeadingsForCsvExport };