@tanstack/ai-sandbox/src/journal-sweep.ts

Provider-agnostic sandbox layer for TanStack AI — run harness adapters inside isolated sandboxes (defineSandbox, defineWorkspace, withSandbox) with a uniform SandboxHandle, workspace bootstrap, policy, and resumable lifecycle.

/**
 * Bound the journal directory: delete the journals nobody will ever read again,
 * and — far more importantly — refuse to delete anything else.
 *
 * `journalCleanupCommand` already deletes ONE run's journal at the moment its
 * `{"__exit":N}` sentinel is observed. That covers every run a host watched to
 * completion and covers nothing else: a run that reaches its sentinel while
 * DETACHED has no host reading its journal, so nothing observes the sentinel and
 * nothing calls the cleanup. Those journals accumulate in
 * {@link DEFAULT_JOURNAL_DIR} until the sandbox dies, which on a `keepAlive`
 * sandbox may be never. This module is the sweep that bounds them, driven from a
 * cron or a reaper rather than from a run.
 *
 * **Why deleting is dangerous, and therefore why almost every branch keeps.**
 * The journal is the ONLY copy of the bytes a successor host needs to replay a
 * run a dead host abandoned mid-flight. Delete a live run's journal and that run
 * becomes unresumable — silently, because the reader will simply deliver nothing.
 * There is no undo and no second copy. So the decision procedure here is not
 * "delete unless I have a reason to keep"; it is the opposite, and every arm that
 * is not a PROVEN-safe deletion keeps:
 *
 * | the store says…                    | action | why |
 * | ---------------------------------- | ------ | --- |
 * | terminal (`isTerminalRunStatus`)   | DELETE | the delivery log, not the journal, is the record |
 * | non-terminal, INCLUDING `'interrupted'` | KEEP | an interrupt-resume continues from it |
 * | nothing (unknown runId)            | KEEP until `orphanTtlMs` | the reader creates the journal BEFORE the record exists |
 * | the lookup threw                   | KEEP | never delete on an unanswered question |
 * | (the name did not decode)          | KEEP | a truncated name decodes to a plausible WRONG runId |
 * | (no mtime listing)                 | KEEP every age-gated entry | cannot age-gate ⇒ cannot expire |
 *
 * Deleting a TERMINAL run's journal is safe because a late takeover of a terminal
 * run aligns against the delivery LOG, not the journal: `align.ts`'s
 * `alignToStoredLog` takes a `StreamDurability` plus an
 * `AsyncIterable<StreamChunk>`, has no `SandboxHandle` and no `JournalPaths` in
 * its signature, and reads the already-delivered prefix with
 * `durability.snapshot()`. It *cannot* read a journal, so removing one cannot
 * break it. A non-zero exit is terminal too — `{"__exit":7}` is as final as
 * `{"__exit":0}`.
 *
 * The unknown-runId arm is the subtle one, and it is why an age gate exists at
 * all. `journalFollowCommand` opens the journal with `: >> file`, which CREATES
 * it; the reader and the run record are written by two independent code paths and
 * nothing orders them. So "a journal exists whose runId the store has never heard
 * of" is the NORMAL state of a run that started moments ago, not an anomaly.
 * Treating unknown as deletable would race every single run start. The journal is
 * therefore kept until it has been untouched for `orphanTtlMs`, which is the only
 * evidence available that no one is writing to it.
 *
 * **The fail-closed trap this module exists to not fall into.** BusyBox `find`
 * prints its "unrecognized option" diagnostic to *stderr* and exits **1 with
 * empty stdout**. A capability probe that ignores the exit code reads that as "no
 * files matched", i.e. "no file is newer than the cutoff" — and code that then
 * concludes "therefore every file is old" **deletes the entire directory**, live
 * runs included. {@link parseJournalMtimeListing} is built to make that
 * impossible: it passes the directory as `stat`'s own first operand as a
 * self-witness and returns `{ kind: 'unavailable' }` when that witness line is
 * absent, never `[]`. This module's whole obligation on that front is to honor
 * `unavailable` as "I cannot age-gate, so I keep" rather than as an empty
 * listing. See the `age-gate-unavailable` reason.
 *
 * **Shell only, never `handle.fs.*`.** On local-process, `fs.*` resolves `/tmp`
 * under the sandbox root while a shell redirect hits the real host `/tmp`, so an
 * `fs.remove` would delete a DIFFERENT path than the one `journaledCommand`
 * wrote — silently doing nothing while reporting success. Every filesystem touch
 * here goes through `handle.process.exec` with a command composed in
 * `journal.ts`.
 */
import { isTerminalRunStatus } from '@tanstack/ai'
import {
  DEFAULT_JOURNAL_DIR,
  decodeJournalRunId,
  journalCleanupCommand,
  journalListCommand,
  journalMtimeListCommand,
  journalPaths,
  parseJournalMtimeListing,
} from './journal'
import type { InternalLogger } from '@tanstack/ai/adapter-internals'
import type { RunStore } from '@tanstack/ai'
import type { SandboxHandle } from './contracts'

/**
 * How long a journal whose runId the store does not know must go untouched
 * before the sweep will delete it.
 *
 * One hour, chosen against what the window actually protects: the gap between a
 * reader creating the journal with `: >> file` and the run record appearing in
 * the store. That gap is milliseconds in the normal case and seconds in the worst
 * case (a slow store, a retried write). An hour is three orders of magnitude of
 * headroom on the race, while still bounding a leaked journal to something a
 * sandbox's disk survives. Erring long is the cheap direction: the cost of too
 * long is bytes, the cost of too short is a destroyed live run.
 */
export const DEFAULT_ORPHAN_TTL_MS = 60 * 60 * 1000

/**
 * Ceiling on deletions per sweep. A cron-driven sweep runs unattended, so a
 * mistake — a store that answers `terminal` for everything, a misconfigured
 * directory — is bounded by this rather than by how many journals happen to
 * exist. The remainder is reported as kept with reason `max-deletes` and picked
 * up by the next sweep.
 */
export const DEFAULT_MAX_DELETES = 200

/** Why {@link pruneJournals} left a journal in place. */
export type KeptJournalReason =
  /** The store answered with a non-terminal status (`'running'`, `'interrupted'`). */
  | 'non-terminal'
  /** The store has never heard of this runId and the journal is still fresh. */
  | 'orphan-too-recent'
  /**
   * The store has never heard of this runId and the age gate could not run at
   * all — {@link parseJournalMtimeListing} returned `unavailable`. THE
   * FAIL-CLOSED ARM: an unavailable listing is not an empty one and says nothing
   * about any file's age.
   */
  | 'age-gate-unavailable'
  /**
   * The age gate ran but reported no mtime for this file, so its age is unknown.
   * (A file created between the two `exec`s, or a name the glob missed.)
   */
  | 'age-gate-missing-entry'
  /** {@link decodeJournalRunId} refused the name (`truncated` or `malformed`). */
  | 'undecodable-name'
  /** The store lookup threw. A question that was not answered is not a licence to delete. */
  | 'store-error'
  /** The `rm` itself failed or exited non-zero. */
  | 'delete-failed'
  /** {@link PruneJournalsOptions.maxDeletes} was already reached this sweep. */
  | 'max-deletes'

/** One journal (or one runId's journal + sidecar) the sweep declined to delete. */
export interface KeptJournal {
  /** The decoded runId; absent exactly when `reason` is `'undecodable-name'`. */
  runId?: string
  /** Every listed filename this entry covers — the journal and its `.err` sidecar. */
  names: Array<string>
  reason: KeptJournalReason
}

/** A non-fatal failure the sweep folded into its result instead of throwing. */
export interface PruneJournalsFailure {
  stage: 'list' | 'mtime-list' | 'store' | 'delete'
  /** Present when the failure is attributable to one run. */
  runId?: string
  message: string
}

/** What one {@link pruneJournals} sweep did. */
export interface PruneJournalsResult {
  /** Filenames `ls -1` reported, before de-duplication by runId. */
  listed: number
  /** Distinct runIds those filenames decoded to. */
  runIds: number
  /** runIds whose journal AND sidecar were deleted, in the order deleted. */
  deleted: Array<string>
  /** Everything left in place, with the reason. */
  kept: Array<KeptJournal>
  /**
   * Whether the mtime age gate was usable this sweep. `'unavailable'` means no
   * orphan could be expired, by design.
   */
  ageGate: 'listed' | 'unavailable'
  failures: Array<PruneJournalsFailure>
}

export interface PruneJournalsOptions {
  /** Sandbox holding the journal directory. Touched only via `process.exec`. */
  handle: SandboxHandle
  /**
   * Run lookup. Only `get` is used: the sweep asks about the runIds it found on
   * disk and never enumerates the store, so no optional `RunStore` method is
   * required of a backend.
   */
  runs: Pick<RunStore, 'get'>
  /** Journal directory. Defaults to {@link DEFAULT_JOURNAL_DIR}. */
  dir?: string
  /** Age-gate reference time. Defaults to `Date.now()`; injectable for tests. */
  now?: number
  /** See {@link DEFAULT_ORPHAN_TTL_MS}. */
  orphanTtlMs?: number
  /** See {@link DEFAULT_MAX_DELETES}. */
  maxDeletes?: number
  logger?: InternalLogger
}

function errorMessage(error: unknown): string {
  return error instanceof Error ? error.message : String(error)
}

/**
 * Group listed filenames by the runId they decode to, so a journal and its
 * `.err` sidecar are ONE decision and ONE `rm`, not two.
 *
 * De-duplication is not a tidiness measure: `journalCleanupCommand` deletes both
 * paths for a runId at once, so iterating raw names would ask the store twice per
 * run and then issue a second `rm` for files the first one already removed —
 * doubling the store load and reporting one run as two deletions.
 */
function groupByRunId(names: Array<string>): {
  byRunId: Map<string, Array<string>>
  undecodable: Array<string>
} {
  const byRunId = new Map<string, Array<string>>()
  const undecodable: Array<string> = []
  for (const name of names) {
    const decoded = decodeJournalRunId(name)
    if (decoded.kind !== 'runId') {
      undecodable.push(name)
      continue
    }
    const existing = byRunId.get(decoded.runId)
    if (existing === undefined) byRunId.set(decoded.runId, [name])
    else existing.push(name)
  }
  return { byRunId, undecodable }
}

/**
 * Sweep the journal directory, deleting only journals whose runs the store
 * reports terminal (plus orphans that have been untouched past `orphanTtlMs`).
 *
 * **Never rejects.** This runs unattended from a cron, where a rejected promise
 * is an unhandled rejection and, worse, hides which journals were and were not
 * swept. Every failure — a listing that errored, a store that threw, an `rm` that
 * exited non-zero — is folded into
 * {@link PruneJournalsResult.failures} and the sweep continues with the entries
 * it can still decide about.
 */
export async function pruneJournals(
  options: PruneJournalsOptions,
): Promise<PruneJournalsResult> {
  const dir = options.dir ?? DEFAULT_JOURNAL_DIR
  const now = options.now ?? Date.now()
  const orphanTtlMs = options.orphanTtlMs ?? DEFAULT_ORPHAN_TTL_MS
  const maxDeletes = options.maxDeletes ?? DEFAULT_MAX_DELETES
  const logger = options.logger

  const deleted: Array<string> = []
  const kept: Array<KeptJournal> = []
  const failures: Array<PruneJournalsFailure> = []

  // `ls -1` is the authoritative name list. It is a SEPARATE command from the
  // mtime listing on purpose: `stat -c` may not exist on the provider's
  // busybox, and a sweep that could not enumerate at all when the age gate is
  // unavailable would never delete the terminal journals it is safe to delete.
  let names: Array<string> = []
  try {
    const listing = await options.handle.process.exec(journalListCommand(dir))
    names = listing.stdout
      .split('\n')
      .map((line) => line.trim())
      .filter((line) => line !== '')
  } catch (error) {
    // Nothing was enumerated, so nothing can be deleted. Report and stop —
    // there is no partial-listing arm, because a partial listing is
    // indistinguishable from a complete one and we only ever DELETE from it.
    failures.push({ stage: 'list', message: errorMessage(error) })
    logger?.warn('journal sweep: listing the journal directory failed', {
      dir,
      error,
    })
    return {
      listed: 0,
      runIds: 0,
      deleted,
      kept,
      ageGate: 'unavailable',
      failures,
    }
  }

  // The age gate. `unavailable` is a first-class outcome, NOT an empty listing:
  // see the module doc's BusyBox `find` trap. It disables orphan expiry for this
  // sweep and disables nothing else.
  let ageGate: 'listed' | 'unavailable' = 'unavailable'
  const mtimes = new Map<string, number>()
  try {
    const probe = await options.handle.process.exec(
      journalMtimeListCommand(dir),
    )
    const parsed = parseJournalMtimeListing(probe.stdout, dir)
    if (parsed.kind === 'listed') {
      ageGate = 'listed'
      for (const entry of parsed.entries) mtimes.set(entry.name, entry.mtimeMs)
    } else {
      logger?.warn(
        'journal sweep: mtime listing unavailable; keeping every orphan',
        { dir },
      )
    }
  } catch (error) {
    failures.push({ stage: 'mtime-list', message: errorMessage(error) })
    logger?.warn('journal sweep: mtime listing failed; keeping every orphan', {
      dir,
      error,
    })
  }

  const { byRunId, undecodable } = groupByRunId(names)

  // Undecodable names are kept unconditionally and without asking the store.
  // A truncated name decodes to a PLAUSIBLE BUT WRONG runId, so consulting the
  // store about it would answer a question about some other run — possibly a
  // live one — and a `terminal` answer would then delete this run's journal.
  for (const name of undecodable) {
    kept.push({ names: [name], reason: 'undecodable-name' })
  }

  const orphanCutoff = now - orphanTtlMs

  for (const [runId, runNames] of byRunId) {
    if (deleted.length >= maxDeletes) {
      kept.push({ runId, names: runNames, reason: 'max-deletes' })
      continue
    }

    let record: Awaited<ReturnType<RunStore['get']>>
    try {
      record = await options.runs.get(runId)
    } catch (error) {
      failures.push({ stage: 'store', runId, message: errorMessage(error) })
      logger?.warn('journal sweep: run lookup failed; keeping the journal', {
        runId,
        error,
      })
      kept.push({ runId, names: runNames, reason: 'store-error' })
      continue
    }

    if (record === null) {
      // Unknown to the store: either the record has not been written yet (the
      // normal case for a run that just started) or it was deleted after the
      // run ended. Only age distinguishes them.
      if (ageGate === 'unavailable') {
        kept.push({ runId, names: runNames, reason: 'age-gate-unavailable' })
        continue
      }
      const observed = runNames.map((name) => mtimes.get(name))
      if (observed.some((mtimeMs) => mtimeMs === undefined)) {
        kept.push({ runId, names: runNames, reason: 'age-gate-missing-entry' })
        continue
      }
      // The NEWEST of the run's files decides: a journal whose sidecar was
      // written a second ago is being written to, whatever the journal's own
      // mtime says.
      const newest = Math.max(...observed.filter(isDefined))
      if (newest > orphanCutoff) {
        kept.push({ runId, names: runNames, reason: 'orphan-too-recent' })
        continue
      }
    } else if (!isTerminalRunStatus(record.status)) {
      // `'interrupted'` lands here, and must: it is a human-in-the-loop PAUSE
      // that interrupt-resume continues from, not an end state.
      kept.push({ runId, names: runNames, reason: 'non-terminal' })
      continue
    }

    // Shell `rm`, never `handle.fs.remove`: module doc, and `journalPaths`
    // re-derives byte-identical paths from the runId alone.
    const command = journalCleanupCommand(journalPaths(runId, dir))
    try {
      const result = await options.handle.process.exec(command)
      if (result.exitCode !== 0) {
        failures.push({
          stage: 'delete',
          runId,
          message: `rm exited ${result.exitCode}`,
        })
        kept.push({ runId, names: runNames, reason: 'delete-failed' })
        continue
      }
    } catch (error) {
      // A failed cleanup must never fail the sweep: the journal is still there
      // and the next sweep will see it again.
      failures.push({ stage: 'delete', runId, message: errorMessage(error) })
      logger?.warn('journal sweep: deleting a journal failed', { runId, error })
      kept.push({ runId, names: runNames, reason: 'delete-failed' })
      continue
    }
    deleted.push(runId)
  }

  logger?.sandbox('journal sweep complete', {
    dir,
    listed: names.length,
    runIds: byRunId.size,
    deleted: deleted.length,
    kept: kept.length,
    ageGate,
  })

  return {
    listed: names.length,
    runIds: byRunId.size,
    deleted,
    kept,
    ageGate,
    failures,
  }
}

/** Narrowing predicate: `Array<number | undefined>` → `Array<number>`. */
function isDefined(value: number | undefined): value is number {
  return value !== undefined
}