@fontoxml/fontoxml-development-tools/src/modules/editor/dev-cms/connectors-cms-standard/stores/DevelopmentCms.js

Development tools for Fonto.

import path from 'path';
import { v4 as uuid } from 'uuid';

import FileSystemStore from './FileSystemStore.js';
import MemoryStore from './MemoryStore.js';
import mergeItems from './mergeItems.js';
import getFileHash from '../getFileHash.js';
import unmergeableRevisionList from './unmergeableRevisionList.js';

/** @typedef {import('../../../src/getAppConfig.js').DevCmsConfig} DevCmsConfig */

/**
 * @typedef  {Object} DevCmsRevision
 *
 * @property {string} id
 * @property {Object} author
 * @property {string} lastModifiedTimestamp
 * @property {string} _editSessionToken
 */

/**
 * File extensions to use for specific fileType requests.
 * This is used to construct the query string in the browse endpoint to retrieve the correct files from Google Drive.
 *
 * @type {Object}
 */
const ASSET_TYPE_BY_EXTENSION = {
	xml: 'document',
	dita: 'document',
	ditamap: 'document',

	template: 'document-template',

	bmp: 'image',
	gif: 'image',
	jpeg: 'image',
	jpg: 'image',
	png: 'image',
	svg: 'image',

	mp3: 'audio',
	wav: 'audio',
	aac: 'audio',
	ogg: 'audio',
};

export class DevCmsFileLock {
	constructor() {
		/** @type {() => void} */
		this._resolve = null;

		/** @type {Promise<void>} */
		this._promise = new Promise((resolve) => {
			this._resolve = resolve;
		});
	}

	release() {
		if (this._resolve) {
			this._resolve();
			this._resolve = null;
		}
	}

	waitForRelease() {
		if (this._resolve) {
			return this._promise;
		}
		return Promise.resolve();
	}
}

/**
 * A file path object pointing to the `history.json` file.
 */
export class DevCmsHistoryFilePath {
	/**
	 * @param {string} filePath
	 */
	constructor(filePath) {
		/** @type {string} */
		this._filePath = filePath;
	}

	getFilePath() {
		return this._filePath;
	}

	/**
	 * @returns {string}
	 */
	getHistoryFilePath() {
		return path.join('.history', this._filePath, 'history.json');
	}
}

/**
 * A file path object pointing to a specific history revision file.
 */
export class DevCmsHistoryRevisionFilePath extends DevCmsHistoryFilePath {
	/**
	 * @param {string} filePath
	 * @param {string} revisionId
	 */
	constructor(filePath, revisionId) {
		super(filePath);

		/** @type {string} */
		this._revisionId = revisionId;
	}

	/**
	 * @returns {string}
	 */
	getHistoryFilePath() {
		return path.join('.history', this._filePath, this._revisionId);
	}
}

export default class DevelopmentCms {
	/**
	 * @param  {DevCmsConfig}  config
	 */
	constructor(config) {
		/** @type {{ [extension: string]: string}} */
		this._assetTypeByExtension = {
			...ASSET_TYPE_BY_EXTENSION,
			...config.assetTypeByExtension,
		};

		/** @type {Map<string, MemoryStore> }} */
		this._memoryStoreById = new Map();
		/** @type {Map<string, NodeJS.Timeout>} */
		this._memoryStoreCleanupById = new Map();

		/** @type {FileSystemStore} */
		this._fileSystem = new FileSystemStore(config, this._assetTypeByExtension);

		/** @type {DevCmsConfig['memoryStoreTtl']} */
		this._memoryStoreTtl = config.memoryStoreTtl;
		/** @type {DevCmsConfig['saveMode']} */
		this._saveMode = config.saveMode;

		/** @type {Map<string, DevCmsFileLock> }} */
		this._locksByFilePath = new Map();
	}

	/**
	 * @param {string} editSessionToken
	 *
	 * @return {MemoryStore}
	 */
	_getOrCreateMemoryStore(editSessionToken) {
		const memoryStoreId =
			this._saveMode === 'shared-memory'
				? 'shared-memory-store-id'
				: editSessionToken;

		let memoryStore = this._memoryStoreById.get(memoryStoreId);
		if (!memoryStore) {
			memoryStore = new MemoryStore(this._assetTypeByExtension);
			this._memoryStoreById.set(memoryStoreId, memoryStore);
		}

		if (this._memoryStoreTtl < 0) {
			return memoryStore;
		}

		const memoryStoreCleanup = this._memoryStoreCleanupById.get(memoryStoreId);
		if (memoryStoreCleanup) {
			clearTimeout(memoryStoreCleanup);
		}

		this._memoryStoreCleanupById.set(
			memoryStoreId,
			setTimeout(() => {
				this._memoryStoreCleanupById.delete(memoryStoreId);
				this._memoryStoreById.delete(memoryStoreId);
			}, this._memoryStoreTtl),
		);

		return memoryStore;
	}

	/**
	 * Get a lock for a file.
	 *
	 * @description
	 * When performing a read-update-write on a file, be sure to use a single lock for the file, to
	 * prevent any other process from modifying the file in the meantime, and the write potentially
	 * overriding those changes.
	 *
	 * WARNING: When handling files in parallel, and you (might) process the same file multiple
	 * times, you need to acquire the lock per parallel process, and not once per file. Otherwise
	 * race conditions might still occur, potentially leading to data loss.
	 *
	 * TIP: When using in a route handler, be sure to use `asyncRouteWithLockCleanupHandler`, which
	 * makes sure the locks are also released on error.
	 *
	 * @example
	 * ```
	 * const fileLock = await devCms.acquireLock(filePath);
	 * const file = await devCms.getRevisionsAndLatestContent(filePath, editSessionToken, fileLock);
	 * const updatedContent = file.content.replace('foo', 'bar');
	 * await devCms.updateFile(filePath, updatedContent, currentSession, fileLock);
	 * fileLock.release();
	 * ```
	 *
	 * @example
	 * ```
	 * return asyncRouteWithLockCleanupHandler(async (acquireLock, req, res) => {
	 * 	// Note using `await acquireLock()` instead of `req.cms.acquireLock()`.
	 * 	const fileLock = await acquireLock(filePath);
	 * 	const file = await devCms.getRevisionsAndLatestContent(filePath, editSessionToken, fileLock);
	 * 	// Note that the lock is also cleaned up on error at the end of the route handler.
	 * 	fileLock.release();
	 * 	res.status(200).send(file.content);
	 * });
	 * ```
	 *
	 * @param {string} filePath
	 *
	 * @return {Promise<DevCmsFileLock>}
	 */
	async acquireLock(filePath) {
		if (!filePath) {
			throw new Error('No path specified for lock.');
		}

		if (typeof filePath !== 'string') {
			throw new Error('Path must be a string.');
		}

		if (!path.relative('.history/', filePath).startsWith('..')) {
			// Having the lock on a file, also means you have a lock on the history file(s).
			throw new Error('Cannot get lock for history file.');
		}

		// Wait until any earlier lock is released.
		while (this._locksByFilePath.has(filePath)) {
			const fileLock = this._locksByFilePath.get(filePath);
			await fileLock?.waitForRelease();
		}

		const fileLock = new DevCmsFileLock();
		this._locksByFilePath.set(filePath, fileLock);

		void fileLock.waitForRelease().then(() => {
			if (this._locksByFilePath.get(filePath) === fileLock) {
				// If the lock is still the active lock, remove it.
				this._locksByFilePath.delete(filePath);
			}
		});

		return fileLock;
	}

	/**
	 * @param {string | DevCmsHistoryFilePath} filePath
	 * @param {DevCmsFileLock}                 fileLock
	 *
	 * @return {Promise<void>}
	 */
	async ensureValidLock(filePath, fileLock) {
		if (!filePath) {
			throw new Error('No path specified for lock.');
		}

		const normalizedFilePath =
			typeof filePath === 'string' ? filePath : filePath.getFilePath();
		if (!normalizedFilePath) {
			throw new Error('No path specified for lock.');
		}

		if (!fileLock) {
			throw new Error('No lock specified.');
		}

		if (this._locksByFilePath.get(normalizedFilePath) !== fileLock) {
			throw new Error('Lock not acquired.');
		}
	}

	/**
	 * @param {string | DevCmsHistoryFilePath} filePath
	 * @param {string}                         editSessionToken
	 * @param {DevCmsFileLock}                 fileLock
	 *
	 * @return {Promise<string | null>}
	 */
	async _getFile(filePath, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const storeFilePath =
			typeof filePath === 'string' ? filePath : filePath.getHistoryFilePath();

		const memoryStore = this._getOrCreateMemoryStore(editSessionToken);
		const file = memoryStore.existsSync(storeFilePath)
			? await memoryStore.load(storeFilePath)
			: await this._fileSystem.load(storeFilePath);

		return file;
	}

	/**
	 * @param {string | DevCmsHistoryFilePath} filePath         path of the file relative to dev-cms/files
	 * @param {*}                              content
	 * @param {string}                         editSessionToken
	 * @param {DevCmsFileLock}                 fileLock
	 *
	 * @return {Promise<void>}
	 */
	async _updateFile(filePath, content, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const storeFilePath =
			typeof filePath === 'string' ? filePath : filePath.getHistoryFilePath();

		switch (this._saveMode) {
			case 'session-memory':
			case 'shared-memory':
				await this._getOrCreateMemoryStore(editSessionToken).save(
					storeFilePath,
					content,
				);
				break;

			default:
				await this._fileSystem.save(storeFilePath, content);
		}
	}

	/**
	 * @param {string | DevCmsHistoryFilePath} filePath
	 * @param {*}                              content
	 * @param {string}                         editSessionToken
	 * @param {DevCmsFileLock}                 fileLock
	 *
	 * @return {Promise<void>}
	 */
	async _createFile(filePath, content, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const storeFilePath =
			typeof filePath === 'string' ? filePath : filePath.getHistoryFilePath();
		if (
			this._getOrCreateMemoryStore(editSessionToken).existsSync(
				storeFilePath,
			) ||
			this._fileSystem.existsSync(storeFilePath)
		) {
			// TODO: Decide what we want to do here. Maybe return false? Or just throw and error?
			const error = new Error('File already exists.');
			error.status = 409;
			throw error;
		}

		await this._updateFile(filePath, content, editSessionToken, fileLock);
	}

	/**
	 * @param {string | DevCmsHistoryFilePath} filePath
	 * @param {string}                         editSessionToken
	 * @param {DevCmsFileLock}                 fileLock
	 *
	 * @return {Promise<void>}
	 */
	async _removeFile(filePath, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const storeFilePath =
			typeof filePath === 'string' ? filePath : filePath.getHistoryFilePath();
		this._getOrCreateMemoryStore(editSessionToken).removeSync(storeFilePath);
		this._fileSystem.removeSync(storeFilePath);
	}

	/**
	 * @param {string}           filePath
	 * @param {DevCmsRevision[]} revisions
	 * @param {string}           editSessionToken
	 * @param {DevCmsFileLock}   fileLock
	 *
	 * @return {Promise<void>}
	 */
	async _updateRevisions(filePath, revisions, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		if (typeof filePath !== 'string') {
			throw new Error('Path must be a string.');
		}

		const historyFilePath = new DevCmsHistoryFilePath(filePath);
		const json = JSON.stringify(revisions, null, '\t');
		await this._updateFile(historyFilePath, json, editSessionToken, fileLock);
	}

	/**
	 * @param {string}         filePath
	 * @param {*}              content
	 * @param {Object}         author
	 * @param {string}         editSessionToken
	 * @param {boolean}        isFromSameSession
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<DevCmsRevision[]>}
	 */
	async _initRevisions(
		filePath,
		content,
		author,
		editSessionToken,
		isFromSameSession,
		fileLock,
	) {
		if (typeof filePath !== 'string') {
			throw new Error('Path must be a string.');
		}

		const initialRevisionId = uuid();
		const initialRevisionFilePath = new DevCmsHistoryRevisionFilePath(
			filePath,
			initialRevisionId,
		);
		await this._createFile(
			initialRevisionFilePath,
			content,
			editSessionToken,
			fileLock,
		);

		const revisions = [
			{
				id: initialRevisionId,
				hash: getFileHash(content),
				author,
				lastModifiedTimestamp: new Date().toISOString(),
				// Non-standard, used to combine saves per session
				_editSessionToken: isFromSameSession ? editSessionToken : uuid(),
			},
		];

		// Force save to memory store, or revision ids between loadHistory call of editor and
		// document history service will never match.
		await this._updateRevisions(
			filePath,
			revisions,
			editSessionToken,
			fileLock,
		);

		return revisions;
	}

	/**
	 * @param {string}         filePath
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<{ revisions: DevCmsRevision[], content: string } | null>}
	 */
	async getRevisionsAndLatestContent(filePath, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const content = await this._getFile(filePath, editSessionToken, fileLock);
		if (!content) {
			return null;
		}

		const historyFilePath = new DevCmsHistoryFilePath(filePath);
		const historyFileJson = await this._getFile(
			historyFilePath,
			editSessionToken,
			fileLock,
		);
		if (historyFileJson) {
			// If there is a history file, load it and check if there is a revision matching the
			// current file content.
			const revisions = JSON.parse(historyFileJson);

			// Either return revision, or create a new revision on the fly and use that as latest.
			const hashInHistory = revisions[0]?.hash;
			const currentContentHash = getFileHash(content);
			if (revisions[0] && hashInHistory === currentContentHash) {
				return { content, revisions };
			}

			const session = {
				editSessionToken: uuid(),
				user: {
					displayName: 'Unknown',
					id: uuid(),
				},
			};
			const revisionId = uuid();
			revisions.unshift({
				id: revisionId,
				hash: currentContentHash,
				author: session.user,
				lastModifiedTimestamp: new Date().toISOString(),
				// Non-standard, used to combine saves per session
				_editSessionToken: session.editSessionToken,
			});

			const revisionFilePath = new DevCmsHistoryRevisionFilePath(
				filePath,
				revisionId,
			);
			await this._updateFile(
				revisionFilePath,
				content,
				editSessionToken,
				fileLock,
			);
			await this._updateRevisions(
				filePath,
				revisions,
				editSessionToken,
				fileLock,
			);

			return { content, revisions };
		}

		// No history for this file found, create one on the fly with only the initial revision
		const author = {
			displayName: 'Unknown',
			id: uuid(),
		};
		const revisions = await this._initRevisions(
			filePath,
			content,
			author,
			editSessionToken,
			false,
			fileLock,
		);
		return { content, revisions };
	}

	/**
	 * @param {string}         filePath
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<DevCmsRevision | null>}
	 */
	async getLatestRevision(filePath, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const revisionsAndLatestContent = await this.getRevisionsAndLatestContent(
			filePath,
			editSessionToken,
			fileLock,
		);
		if (!revisionsAndLatestContent) {
			return null;
		}

		return revisionsAndLatestContent.revisions[0];
	}

	/**
	 * @param {string}         filePath
	 * @param {string}         revisionId
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<string | null>}
	 */
	async getFileByRevisionId(filePath, revisionId, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		if (!this.existsSync(filePath, editSessionToken)) {
			return null;
		}

		// TODO: Should we check if the revision id exists in the history file?

		// Having the lock on a file, also means you have a lock on the revision file.
		const revisionFilePath = new DevCmsHistoryRevisionFilePath(
			filePath,
			revisionId,
		);
		return this._getFile(revisionFilePath, editSessionToken, fileLock);
	}

	/**
	 * @param {string}         filePath
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<{ content: string, revisionId: string } | null>}
	 */
	async getFileAndLatestRevisionId(filePath, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const revisionsAndLatestContent = await this.getRevisionsAndLatestContent(
			filePath,
			editSessionToken,
			fileLock,
		);
		if (!revisionsAndLatestContent) {
			return null;
		}

		return {
			content: revisionsAndLatestContent.content,
			revisionId: revisionsAndLatestContent.revisions[0].id,
		};
	}

	/**
	 * @param {string}         filePath
	 * @param {*}              content
	 * @param {Object}         author
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<string>} The revision id
	 */
	async updateFile(filePath, content, author, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const revisionsAndLatestContent = await this.getRevisionsAndLatestContent(
			filePath,
			editSessionToken,
			fileLock,
		);
		if (!revisionsAndLatestContent) {
			// TODO: Should we throw a generic error here? Or return something else?
			const error = new Error('File does not exist.');
			error.status = 404;
			throw error;
		}

		await this._updateFile(filePath, content, editSessionToken, fileLock);

		// "Update" latest revision if this is the same session, unless it's in the unmergeable revision list.
		const revisions = revisionsAndLatestContent.revisions;
		const previousRevision = revisions[0];
		if (
			previousRevision &&
			// TODO: This list should be pre-filled with all created context revisions of existing annotations.
			!unmergeableRevisionList.has(previousRevision.id) &&
			previousRevision._editSessionToken === editSessionToken
		) {
			await this._removeFile(
				new DevCmsHistoryRevisionFilePath(filePath, previousRevision.id),
				editSessionToken,
				fileLock,
			);
			revisions.shift();
		}

		const revisionId = uuid();
		revisions.unshift({
			id: revisionId,
			hash: getFileHash(content),
			author,
			lastModifiedTimestamp: new Date().toISOString(),
			// Non-standard, used to combine saves per session
			_editSessionToken: editSessionToken,
		});

		const revisionFilePath = new DevCmsHistoryRevisionFilePath(
			filePath,
			revisionId,
		);
		await this._updateFile(
			revisionFilePath,
			content,
			editSessionToken,
			fileLock,
		);
		await this._updateRevisions(
			filePath,
			revisions,
			editSessionToken,
			fileLock,
		);

		return revisionId;
	}

	/**
	 * @param {string}         filePath
	 * @param {*}              content
	 * @param {Object}         author
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @returns {Promise<string>} The revision id
	 */
	async createFile(filePath, content, author, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		await this._createFile(filePath, content, editSessionToken, fileLock);
		const revisions = await this._initRevisions(
			filePath,
			content,
			author,
			editSessionToken,
			true,
			fileLock,
		);
		return revisions[0].id;
	}

	/**
	 * NOTE: Only use this for non document files.
	 *
	 * @param {string} filePath
	 * @param {string} editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<string | null>}
	 */
	async getFileWithoutHistory(filePath, editSessionToken, fileLock) {
		await this.ensureValidLock(filePath, fileLock);

		const content = await this._getFile(filePath, editSessionToken, fileLock);
		if (!content) {
			return null;
		}

		return content;
	}

	/**
	 * NOTE: Only use this for non document files.
	 *
	 * @param {string}         filePath
	 * @param {*}              content
	 * @param {string}         editSessionToken
	 * @param {DevCmsFileLock} fileLock
	 *
	 * @return {Promise<void>}
	 */
	async updateOrCreateFileWithoutHistory(
		filePath,
		content,
		editSessionToken,
		fileLock,
	) {
		await this.ensureValidLock(filePath, fileLock);

		// TODO: Maybe split into separate update and create methods?

		await this._updateFile(filePath, content, editSessionToken, fileLock);
	}

	/**
	 * @param {string} filePath         path of the file relative to dev-cms/files
	 * @param {string} editSessionToken
	 *
	 * @return  {boolean}
	 */
	// TODO: Remove this method, and use load instead, checking for error.status === 404 / null??
	existsSync(filePath, editSessionToken) {
		return (
			this._getOrCreateMemoryStore(editSessionToken).existsSync(filePath) ||
			this._fileSystem.existsSync(filePath)
		);
	}

	/**
	 * @param {string} filePath         path of the file relative to dev-cms/files
	 * @param {string} editSessionToken
	 *
	 * @return {string | null} the resolved absolute path of the file on disk. Does not work for files in memory.
	 */
	getPathInFilesystemSync(filePath, _editSessionToken) {
		return this._fileSystem.getPathSync(filePath);
	}

	/**
	 * @param {string | undefined} folderPath       path of the folder relative to dev-cms/files
	 * @param {string}             editSessionToken
	 *
	 * @return {Object[] | null} items in folder, combined from all stores, or null if the folder does not exist.
	 */
	listSync(folderPath, editSessionToken) {
		folderPath = folderPath || '';

		const fileSystemItems = this._fileSystem.listSync(folderPath);
		const memoryItems =
			this._getOrCreateMemoryStore(editSessionToken).listSync(folderPath);

		if (!fileSystemItems && !memoryItems) {
			return null;
		}

		return mergeItems(fileSystemItems, memoryItems);
	}

	/**
	 * @description
	 * For unit testing only.
	 */
	destroy() {
		for (const memoryStoreId of this._memoryStoreById.keys()) {
			this._memoryStoreById.delete(memoryStoreId);
			const memoryStoreCleanup =
				this._memoryStoreCleanupById.get(memoryStoreId);
			if (memoryStoreCleanup) {
				clearTimeout(memoryStoreCleanup);
				this._memoryStoreCleanupById.delete(memoryStoreId);
			}
		}
	}
}