@zenfs/core/dist/backends/cow.js

A filesystem, anywhere

import { EventEmitter } from 'eventemitter3';
import { withErrno } from 'kerium';
import { debug, err, warn } from 'kerium/log';
import { canary } from 'utilium';
import { resolveMountConfig } from '../config.js';
import { FileSystem } from '../internal/filesystem.js';
import { isDirectory } from '../internal/inode.js';
import { dirname, join } from '../path.js';
const journalOperations = new Set(['delete']);
/** Because TS doesn't work right w/o it */
function isJournalOp(op) {
    return journalOperations.has(op);
}
const maxOpLength = Math.max(...journalOperations.values().map(op => op.length));
const journalMagicString = '#journal@v0\n';
/**
 * Tracks various operations for the CoW backend
 * @category Internals
 * @internal
 */
export class Journal extends EventEmitter {
    entries = [];
    toString() {
        return journalMagicString + this.entries.map(entry => `${entry.op.padEnd(maxOpLength)} ${entry.path}`).join('\n');
    }
    /**
     * Parse a journal from a string
     */
    fromString(value) {
        if (!value.startsWith(journalMagicString))
            throw err(withErrno('EINVAL', 'Invalid journal contents, refusing to parse'));
        for (const line of value.split('\n')) {
            if (line.startsWith('#'))
                continue; // ignore comments
            const [op, path] = line.split(/\s+/);
            if (!isJournalOp(op)) {
                warn('Unknown operation in journal (skipping): ' + op);
                continue;
            }
            this.entries.push({ op, path });
        }
        return this;
    }
    add(op, path) {
        this.entries.push({ op, path });
        this.emit('update', op, path);
        this.emit(op, path);
    }
    has(op, path) {
        const test = JSON.stringify({ op, path });
        for (const entry of this.entries)
            if (JSON.stringify(entry) === test)
                return true;
        return false;
    }
    isDeleted(path) {
        let deleted = false;
        for (const entry of this.entries) {
            if (entry.path != path)
                continue;
            switch (entry.op) {
                case 'delete':
                    deleted = true;
            }
        }
        return deleted;
    }
}
/**
 * Using a readable file system as a base, writes are done to a writable file system.
 * @internal
 * @category Internals
 */
export class CopyOnWriteFS extends FileSystem {
    readable;
    writable;
    journal;
    async ready() {
        await this.readable.ready();
        await this.writable.ready();
    }
    readySync() {
        this.readable.readySync();
        this.writable.readySync();
    }
    constructor(
    /** The file system that initially populates this file system. */
    readable, 
    /** The file system to write modified files to. */
    writable, 
    /** The journal to use for persisting deletions */
    journal = new Journal()) {
        super(0x62756c6c, readable.name);
        this.readable = readable;
        this.writable = writable;
        this.journal = journal;
        if (writable.attributes.has('no_write')) {
            throw err(withErrno('EINVAL', 'Writable file system can not be written to'));
        }
        readable.attributes.set('no_write');
    }
    isDeleted(path) {
        return this.journal.isDeleted(path);
    }
    /**
     * @todo Consider trying to track information on the writable as well
     */
    usage() {
        return this.readable.usage();
    }
    async sync() {
        await this.writable.sync();
    }
    syncSync() {
        this.writable.syncSync();
    }
    async read(path, buffer, offset, end) {
        return (await this.writable.exists(path))
            ? await this.writable.read(path, buffer, offset, end)
            : await this.readable.read(path, buffer, offset, end);
    }
    readSync(path, buffer, offset, end) {
        return this.writable.existsSync(path) ? this.writable.readSync(path, buffer, offset, end) : this.readable.readSync(path, buffer, offset, end);
    }
    async write(path, buffer, offset) {
        await this.copyForWrite(path);
        return await this.writable.write(path, buffer, offset);
    }
    writeSync(path, buffer, offset) {
        this.copyForWriteSync(path);
        return this.writable.writeSync(path, buffer, offset);
    }
    async rename(oldPath, newPath) {
        await this.copyForWrite(oldPath);
        try {
            await this.writable.rename(oldPath, newPath);
        }
        catch {
            if (this.isDeleted(oldPath))
                throw withErrno('ENOENT');
        }
    }
    renameSync(oldPath, newPath) {
        this.copyForWriteSync(oldPath);
        try {
            this.writable.renameSync(oldPath, newPath);
        }
        catch {
            if (this.isDeleted(oldPath))
                throw withErrno('ENOENT');
        }
    }
    async stat(path) {
        try {
            return await this.writable.stat(path);
        }
        catch {
            if (this.isDeleted(path))
                throw withErrno('ENOENT');
            return await this.readable.stat(path);
        }
    }
    statSync(path) {
        try {
            return this.writable.statSync(path);
        }
        catch {
            if (this.isDeleted(path))
                throw withErrno('ENOENT');
            return this.readable.statSync(path);
        }
    }
    async touch(path, metadata) {
        await this.copyForWrite(path);
        await this.writable.touch(path, metadata);
    }
    touchSync(path, metadata) {
        this.copyForWriteSync(path);
        this.writable.touchSync(path, metadata);
    }
    async createFile(path, options) {
        await this.createParentDirectories(path);
        return await this.writable.createFile(path, options);
    }
    createFileSync(path, options) {
        this.createParentDirectoriesSync(path);
        return this.writable.createFileSync(path, options);
    }
    async link(srcpath, dstpath) {
        await this.copyForWrite(srcpath);
        await this.writable.link(srcpath, dstpath);
    }
    linkSync(srcpath, dstpath) {
        this.copyForWriteSync(srcpath);
        this.writable.linkSync(srcpath, dstpath);
    }
    async unlink(path) {
        if (!(await this.exists(path)))
            throw withErrno('ENOENT');
        if (await this.writable.exists(path)) {
            await this.writable.unlink(path);
        }
        // if it still exists add to the delete log
        if (await this.exists(path)) {
            this.journal.add('delete', path);
        }
    }
    unlinkSync(path) {
        if (!this.existsSync(path))
            throw withErrno('ENOENT');
        if (this.writable.existsSync(path)) {
            this.writable.unlinkSync(path);
        }
        // if it still exists add to the delete log
        if (this.existsSync(path)) {
            this.journal.add('delete', path);
        }
    }
    async rmdir(path) {
        if (!(await this.exists(path)))
            throw withErrno('ENOENT');
        if (await this.writable.exists(path)) {
            await this.writable.rmdir(path);
        }
        if (!(await this.exists(path))) {
            return;
        }
        // Check if directory is empty.
        if ((await this.readdir(path)).length)
            throw withErrno('ENOTEMPTY');
        this.journal.add('delete', path);
    }
    rmdirSync(path) {
        if (!this.existsSync(path))
            throw withErrno('ENOENT');
        if (this.writable.existsSync(path)) {
            this.writable.rmdirSync(path);
        }
        if (!this.existsSync(path)) {
            return;
        }
        // Check if directory is empty.
        if (this.readdirSync(path).length)
            throw withErrno('ENOTEMPTY');
        this.journal.add('delete', path);
    }
    async mkdir(path, options) {
        if (await this.exists(path))
            throw withErrno('EEXIST');
        await this.createParentDirectories(path);
        return await this.writable.mkdir(path, options);
    }
    mkdirSync(path, options) {
        if (this.existsSync(path))
            throw withErrno('EEXIST');
        this.createParentDirectoriesSync(path);
        return this.writable.mkdirSync(path, options);
    }
    async readdir(path) {
        if (this.isDeleted(path) || !(await this.exists(path)))
            throw withErrno('ENOENT');
        const entries = (await this.readable.exists(path)) ? await this.readable.readdir(path) : [];
        if (await this.writable.exists(path))
            for (const entry of await this.writable.readdir(path)) {
                if (!entries.includes(entry))
                    entries.push(entry);
            }
        return entries.filter(entry => !this.isDeleted(join(path, entry)));
    }
    readdirSync(path) {
        if (this.isDeleted(path) || !this.existsSync(path))
            throw withErrno('ENOENT');
        const entries = this.readable.existsSync(path) ? this.readable.readdirSync(path) : [];
        if (this.writable.existsSync(path))
            for (const entry of this.writable.readdirSync(path)) {
                if (!entries.includes(entry))
                    entries.push(entry);
            }
        return entries.filter(entry => !this.isDeleted(join(path, entry)));
    }
    streamRead(path, options) {
        return this.writable.existsSync(path) ? this.writable.streamRead(path, options) : this.readable.streamRead(path, options);
    }
    streamWrite(path, options) {
        this.copyForWriteSync(path);
        return this.writable.streamWrite(path, options);
    }
    /**
     * Create the needed parent directories on the writable storage should they not exist.
     * Use modes from the read-only storage.
     */
    createParentDirectoriesSync(path) {
        const toCreate = [];
        const silence = canary(withErrno('EDEADLK'));
        for (let parent = dirname(path); !this.writable.existsSync(parent); parent = dirname(parent)) {
            toCreate.push(parent);
        }
        silence();
        if (toCreate.length)
            debug('COW: Creating parent directories: ' + toCreate.join(', '));
        for (const path of toCreate.reverse()) {
            this.writable.mkdirSync(path, this.statSync(path));
        }
    }
    /**
     * Create the needed parent directories on the writable storage should they not exist.
     * Use modes from the read-only storage.
     */
    async createParentDirectories(path) {
        const toCreate = [];
        const silence = canary(withErrno('EDEADLK', path));
        for (let parent = dirname(path); !(await this.writable.exists(parent)); parent = dirname(parent)) {
            toCreate.push(parent);
        }
        silence();
        if (toCreate.length)
            debug('COW: Creating parent directories: ' + toCreate.join(', '));
        for (const path of toCreate.reverse()) {
            await this.writable.mkdir(path, await this.stat(path));
        }
    }
    /**
     * Helper function:
     * - Ensures p is on writable before proceeding. Throws an error if it doesn't exist.
     * - Calls f to perform operation on writable.
     */
    copyForWriteSync(path) {
        if (!this.existsSync(path))
            throw withErrno('ENOENT');
        if (!this.writable.existsSync(dirname(path))) {
            this.createParentDirectoriesSync(path);
        }
        if (!this.writable.existsSync(path)) {
            this.copyToWritableSync(path);
        }
    }
    async copyForWrite(path) {
        if (!(await this.exists(path)))
            throw withErrno('ENOENT');
        if (!(await this.writable.exists(dirname(path)))) {
            await this.createParentDirectories(path);
        }
        if (!(await this.writable.exists(path))) {
            return this.copyToWritable(path);
        }
    }
    /**
     * Copy from readable to writable storage.
     * PRECONDITION: File does not exist on writable storage.
     */
    copyToWritableSync(path) {
        const stats = this.readable.statSync(path);
        if (isDirectory(stats)) {
            this.writable.mkdirSync(path, stats);
            for (const k of this.readable.readdirSync(path)) {
                this.copyToWritableSync(join(path, k));
            }
            return;
        }
        const data = new Uint8Array(stats.size);
        this.readable.readSync(path, data, 0, data.byteLength);
        this.writable.createFileSync(path, stats);
        this.writable.touchSync(path, stats);
        this.writable.writeSync(path, data, 0);
    }
    async copyToWritable(path) {
        const stats = await this.readable.stat(path);
        if (isDirectory(stats)) {
            await this.writable.mkdir(path, stats);
            for (const k of await this.readable.readdir(path)) {
                await this.copyToWritable(join(path, k));
            }
            return;
        }
        const data = new Uint8Array(stats.size);
        await this.readable.read(path, data, 0, stats.size);
        await this.writable.createFile(path, stats);
        await this.writable.touch(path, stats);
        await this.writable.write(path, data, 0);
    }
}
const _CopyOnWrite = {
    name: 'CopyOnWrite',
    options: {
        writable: { type: 'object', required: true },
        readable: { type: 'object', required: true },
        journal: { type: 'object', required: false },
    },
    async create(options) {
        const readable = await resolveMountConfig(options.readable);
        const writable = await resolveMountConfig(options.writable);
        return new CopyOnWriteFS(readable, writable, options.journal);
    },
};
/**
 * Overlay makes a read-only filesystem writable by storing writes on a second, writable file system.
 * Deletes are persisted via metadata stored on the writable file system.
 * @category Backends and Configuration
 * @internal
 */
export const CopyOnWrite = _CopyOnWrite;