convex/src/bundler/fs.ts

Client for the Convex Cloud

import { Dirent, Mode, Stats } from "fs";
import stdFs from "fs";
import path from "path";

export interface Filesystem {
  listDir(dirPath: string): Dirent[];
  exists(path: string): boolean;
  stat(path: string): Stats;
  readUtf8File(path: string): string;
  access(path: string): void;

  writeUtf8File(path: string, contents: string, mode?: Mode): void;
  mkdir(path: string, options?: { allowExisting?: boolean }): void;
  rm(path: string, options?: { force?: boolean; recursive?: boolean }): void;
  unlink(path: string): void;

  registerPath(path: string, st: Stats | null): void;
  invalidate(): void;
}

// Use `nodeFs` when you just want to read and write to the local filesystem
// and don't care about collecting the paths touched. One-off commands
// should use the singleton `nodeFs`.
class NodeFs implements Filesystem {
  listDir(dirPath: string) {
    return stdFs.readdirSync(dirPath, { withFileTypes: true });
  }
  exists(path: string) {
    try {
      stdFs.statSync(path);
      return true;
    } catch (e: any) {
      if (e.code === "ENOENT") {
        return false;
      }
      throw e;
    }
  }
  stat(path: string) {
    return stdFs.statSync(path);
  }
  readUtf8File(path: string) {
    return stdFs.readFileSync(path, { encoding: "utf-8" });
  }
  access(path: string) {
    return stdFs.accessSync(path);
  }
  writeUtf8File(path: string, contents: string, mode?: Mode) {
    const fd = stdFs.openSync(path, "w", mode);
    try {
      stdFs.writeFileSync(fd, contents, { encoding: "utf-8" });
      stdFs.fsyncSync(fd);
    } finally {
      stdFs.closeSync(fd);
    }
  }
  mkdir(path: string, options?: { allowExisting?: boolean }) {
    try {
      stdFs.mkdirSync(path);
    } catch (e: any) {
      if (options?.allowExisting && e.code == "EEXIST") {
        return;
      }
      throw e;
    }
  }
  rm(path: string, options?: { force?: boolean; recursive?: boolean }) {
    stdFs.rmSync(path, options);
  }
  unlink(path: string) {
    return stdFs.unlinkSync(path);
  }

  registerPath(_path: string, _st: Stats | null) {
    // The node filesystem doesn't track reads, so we don't need to do anything here.
  }
  invalidate() {
    // We don't track invalidations for the node filesystem either.
  }
}
export const nodeFs: Filesystem = new NodeFs();

// Filesystem implementation that records all paths observed. This is useful
// for implementing continuous watch commands that need to manage a filesystem
// watcher and know when a command's inputs were invalidated.
export class RecordingFs implements Filesystem {
  // Absolute path -> Set of observed child names
  private observedDirectories: Map<string, Set<string>> = new Map();

  // Absolute path -> observed stat (or null if observed nonexistent)
  private observedFiles: Map<string, Stats | null> = new Map();

  // Have we noticed that files have changed while recording?
  private invalidated = false;

  private traceEvents: boolean;

  constructor(traceEvents: boolean) {
    this.traceEvents = traceEvents;
  }

  listDir(dirPath: string): Dirent[] {
    const absDirPath = path.resolve(dirPath);

    // Register observing the directory itself.
    const dirSt = nodeFs.stat(absDirPath);
    this.registerNormalized(absDirPath, dirSt);

    // List the directory and register observing all of its children.
    const entries = nodeFs.listDir(dirPath);
    for (const entry of entries) {
      const childPath = path.join(absDirPath, entry.name);
      const childSt = nodeFs.stat(childPath);
      this.registerPath(childPath, childSt);
    }

    // Register observing the directory's children.
    const observedNames = new Set(entries.map(e => e.name));
    const existingNames = this.observedDirectories.get(absDirPath);
    if (existingNames) {
      if (!setsEqual(observedNames, existingNames)) {
        this.invalidated = true;
      }
    }
    this.observedDirectories.set(absDirPath, observedNames);

    return entries;
  }

  exists(path: string): boolean {
    try {
      const st = nodeFs.stat(path);
      this.registerPath(path, st);
      return true;
    } catch (err: any) {
      if (err.code === "ENOENT") {
        this.registerPath(path, null);
        return false;
      }
      throw err;
    }
  }
  stat(path: string): Stats {
    try {
      const st = nodeFs.stat(path);
      this.registerPath(path, st);
      return st;
    } catch (err: any) {
      if (err.code === "ENOENT") {
        this.registerPath(path, null);
      }
      throw err;
    }
  }
  readUtf8File(path: string): string {
    try {
      const st = nodeFs.stat(path);
      this.registerPath(path, st);
      return nodeFs.readUtf8File(path);
    } catch (err: any) {
      if (err.code === "ENOENT") {
        this.registerPath(path, null);
      }
      throw err;
    }
  }
  access(path: string) {
    try {
      const st = nodeFs.stat(path);
      this.registerPath(path, st);
      return nodeFs.access(path);
    } catch (err: any) {
      if (err.code === "ENOENT") {
        this.registerPath(path, null);
      }
      throw err;
    }
  }

  writeUtf8File(filePath: string, contents: string, mode?: Mode) {
    const absPath = path.resolve(filePath);

    nodeFs.writeUtf8File(filePath, contents, mode);

    // Stat the file after writing and make it our expected observation. If we read the file after
    // writing it and it doesn't match this stat (implying a subsequent write), we'll invalidate
    // the current reader.
    const newSt = nodeFs.stat(absPath);

    // Skip invalidation checking since we don't want to conflict if we previously read this file.
    this.observedFiles.set(absPath, newSt);

    // If we observed the parent, add the new file to its expected directory entries.
    const parentPath = path.resolve(path.dirname(absPath));
    const observedParent = this.observedDirectories.get(parentPath);
    if (observedParent !== undefined) {
      observedParent.add(path.basename(absPath));
    }
  }
  mkdir(dirPath: string, options?: { allowExisting?: boolean }) {
    const absPath = path.resolve(dirPath);
    try {
      stdFs.mkdirSync(absPath);
    } catch (e: any) {
      if (options?.allowExisting && e.code === "EEXIST") {
        const st = nodeFs.stat(absPath);
        this.registerNormalized(absPath, st);
        return;
      }
      throw e;
    }
    // Stat the directory we just created.
    const newSt = nodeFs.stat(absPath);
    this.observedFiles.set(absPath, newSt);

    // If we observed the parent, add our newly created file.
    const parentPath = path.resolve(path.dirname(absPath));
    const observedParent = this.observedDirectories.get(parentPath);
    if (observedParent !== undefined) {
      observedParent.add(path.basename(absPath));
    }
  }
  rm(entityPath: string, options?: { force?: boolean; recursive?: boolean }) {
    const absPath = path.resolve(entityPath);

    // Handle `options.recursive` manually so that we correctly update our observations.
    if (
      options?.recursive &&
      this.exists(absPath) &&
      this.stat(absPath).isDirectory()
    ) {
      const entries = this.listDir(entityPath);
      for (const entry of entries) {
        this.rm(path.join(absPath, entry.name), options);
      }
    }

    stdFs.rmSync(absPath, options);

    // Expect this file/directory to be gone.
    this.observedFiles.set(absPath, null);

    // Unlink it from our parent if observed.
    const parentPath = path.resolve(path.dirname(absPath));
    const observedParent = this.observedDirectories.get(parentPath);
    if (observedParent !== undefined) {
      observedParent.delete(path.basename(absPath));
    }
  }
  unlink(filePath: string) {
    const absPath = path.resolve(filePath);
    stdFs.unlinkSync(absPath);

    // Expect this file to be gone.
    this.observedFiles.set(absPath, null);

    // Unlink it from our parent if observed.
    const parentPath = path.resolve(path.dirname(absPath));
    const observedParent = this.observedDirectories.get(parentPath);
    if (observedParent !== undefined) {
      observedParent.delete(path.basename(absPath));
    }
  }

  registerPath(p: string, st: Stats | null) {
    const absPath = path.resolve(p);
    this.registerNormalized(absPath, st);
  }

  invalidate() {
    this.invalidated = true;
  }

  private registerNormalized(p: string, observed: Stats | null): void {
    const existing = this.observedFiles.get(p);
    if (existing !== undefined) {
      const stMatch = stMatches(observed, existing);
      if (!stMatch.matches) {
        if (this.traceEvents) {
          console.log(
            "Invalidating due to st mismatch",
            p,
            observed,
            existing,
            stMatch.reason
          );
        }
        this.invalidated = true;
      }
    }
    this.observedFiles.set(p, observed);
  }

  finalize(): Observations | "invalidated" {
    if (this.invalidated) {
      return "invalidated";
    }
    return new Observations(this.observedDirectories, this.observedFiles);
  }
}

export type WatchEvent = {
  name: "add" | "addDir" | "change" | "unlink" | "unlinkDir";
  absPath: string;
};

export class Observations {
  directories: Map<string, Set<string>>;
  files: Map<string, Stats | null>;

  constructor(
    directories: Map<string, Set<string>>,
    files: Map<string, Stats | null>
  ) {
    this.directories = directories;
    this.files = files;
  }

  paths(): string[] {
    const out = [];
    for (const path of this.directories.keys()) {
      out.push(path);
    }
    for (const path of this.files.keys()) {
      out.push(path);
    }
    return out;
  }

  overlaps({
    absPath,
  }: WatchEvent): { overlaps: false } | { overlaps: true; reason: string } {
    let currentSt: null | Stats;
    try {
      currentSt = nodeFs.stat(absPath);
    } catch (e: any) {
      if (e.code === "ENOENT") {
        currentSt = null;
      } else {
        throw e;
      }
    }

    // First, check to see if we observed `absPath` as a file.
    const observedSt = this.files.get(absPath);
    if (observedSt !== undefined) {
      const stMatch = stMatches(observedSt, currentSt);
      if (!stMatch.matches) {
        const reason = `modified (${stMatch.reason})`;
        return { overlaps: true, reason };
      }
    }

    // Second, check if we listed the directory this file is in.
    const parentPath = path.resolve(path.dirname(absPath));
    const observedParent = this.directories.get(parentPath);
    if (observedParent !== undefined) {
      const filename = path.basename(absPath);

      // If the file is gone now, but we observed it in its directory, then
      // it was deleted.
      if (currentSt === null && observedParent.has(filename)) {
        return { overlaps: true, reason: "deleted" };
      }

      // If the file exists now, but we didn't see it when listing its directory,
      // then it was added.
      if (currentSt !== null && !observedParent.has(filename)) {
        return { overlaps: true, reason: "added" };
      }
    }

    return { overlaps: false };
  }
}

function setsEqual<T>(a: Set<T>, b: Set<T>): boolean {
  if (a.size !== b.size) {
    return false;
  }
  for (const elem of a.keys()) {
    if (!b.has(elem)) {
      return false;
    }
  }
  return true;
}

export function stMatches(
  a: Stats | null,
  b: Stats | null
): { matches: true } | { matches: false; reason: string } {
  if (a === null && b === null) {
    return { matches: true };
  }
  if (a !== null && b !== null) {
    if (a.dev !== b.dev) {
      return { matches: false, reason: "device boundary" };
    }
    if (a.isFile() || b.isFile()) {
      if (!a.isFile() || !b.isFile()) {
        return { matches: false, reason: "file type" };
      }
      if (a.ino !== b.ino) {
        return {
          matches: false,
          reason: `file inode (${a.ino} vs. ${b.ino})`,
        };
      }
      if (a.size !== b.size) {
        return {
          matches: false,
          reason: `file size (${a.size} vs. ${b.size})`,
        };
      }
      if (a.mtimeMs !== b.mtimeMs) {
        return {
          matches: false,
          reason: `file mtime (${a.mtimeMs} vs. ${b.mtimeMs})`,
        };
      }
      return { matches: true };
    }
    if (a.isDirectory() || b.isDirectory()) {
      if (!b.isDirectory() || !b.isDirectory()) {
        return { matches: false, reason: "dir file type" };
      }
      if (a.ino !== b.ino) {
        return {
          matches: false,
          reason: `dir inode (${a.ino} vs. ${b.ino})`,
        };
      }
      return { matches: true };
    }
    // If we have something other than a file or directory, just compare inodes.
    if (a.ino !== b.ino) {
      return {
        matches: false,
        reason: `special inode (${a.ino} vs. ${b.ino})`,
      };
    }
    return { matches: true };
  }
  return { matches: false, reason: "deleted mismatch" };
}