@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
740 lines (692 loc) • 29.7 kB
text/typescript
import { createHash } from "node:crypto";
import { closeSync, fstatSync, openSync, readSync } from "node:fs";
import {
currentUsageLogRevision,
exactFileIdentity,
normalizePersistedUsageRow,
usageLogIdentityKey,
usageLogPath,
usageLogRevisionKey,
type PersistedUsageEntry,
type UsageLogRevision,
} from "./log";
export const USAGE_LEDGER_READ_CHUNK_BYTES = 1024 * 1024;
/** First window tried when reading one indexed row; most rows fit many times over. */
export const USAGE_LEDGER_ROW_READ_WINDOW_BYTES = 64 * 1024;
// Normalized writer rows carry a <=16 KiB route trace and <=500-character captured
// upstream error, so 1 MiB leaves wide headroom even for an extreme multi-attempt row.
// Hand-edited rows can still exceed it; those are reported separately instead of making
// one unterminated line an unbounded allocation.
export const USAGE_LEDGER_MAX_LINE_BYTES = 1024 * 1024;
export const USAGE_LEDGER_BOUNDARY_DIGEST_BYTES = 64 * 1024;
const MAX_SCAN_SLICE_MS = 8;
export interface ScanUsageLedgerOptions {
signal?: AbortSignal;
/**
* Receives each complete row and the absolute byte offset it starts at, so a derived
* index can address the same row later without re-reading the whole ledger.
*/
onEntry: (entry: PersistedUsageEntry, rowStartBytes: number) => void;
/** Absolute LF boundary returned by a previous scan. Defaults to byte zero. */
startAtBytes?: number;
/** Stable path/dev/ino/birthtime identity; required when startAtBytes is nonzero. */
expectedIdentityKey?: string;
/** Trailing digest at the previous boundary; required when startAtBytes is nonzero. */
expectedProcessedThroughDigest?: string;
/** Test seam for forcing byte boundaries; production always uses the 1 MiB default. */
chunkBytes?: number;
}
export interface UsageLedgerScanResult {
/** Revision whose EOF was captured when the scan opened the ledger. */
revision: UsageLogRevision | null;
parsedRows: number;
/** LF-complete malformed/schema-invalid rows plus a non-empty bounded torn suffix. */
invalidRows: number;
/** Rows skipped after exceeding USAGE_LEDGER_MAX_LINE_BYTES. */
oversizedRows: number;
bytesRead: number;
/** Absolute byte offset immediately after the last handled LF. */
processedThroughBytes: number;
/** SHA-256 over at most the last 64 KiB ending at processedThroughBytes. */
processedThroughDigest: string;
}
export type UsageLedgerRebuildReason =
| "identity_mismatch"
| "shrink"
| "boundary_mismatch"
| "content_changed";
export class UsageLedgerRebuildRequiredError extends Error {
readonly code = "usage_ledger_rebuild_required";
constructor(readonly reason: UsageLedgerRebuildReason) {
super(`usage ledger rebuild required: ${reason}`);
this.name = "UsageLedgerRebuildRequiredError";
}
}
function revisionFromStat(
path: string,
fd: number,
stat: ReturnType<typeof fstatSync>,
): UsageLogRevision {
if (!stat.isFile()) throw new Error("usage log is not a regular file");
return {
path,
...exactFileIdentity(fd),
birthtimeMs: Number(stat.birthtimeMs),
size: Number(stat.size),
mtimeMs: Number(stat.mtimeMs),
ctimeMs: Number(stat.ctimeMs),
};
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (!signal?.aborted) return;
throw signal.reason ?? new Error("usage ledger scan aborted");
}
function isMissingFileError(error: unknown): boolean {
return error !== null
&& typeof error === "object"
&& "code" in error
&& error.code === "ENOENT";
}
function isJsonWhitespace(bytes: Buffer, length: number): boolean {
for (let index = 0; index < length; index += 1) {
const byte = bytes[index];
if (byte !== 0x20 && byte !== 0x09 && byte !== 0x0d) return false;
}
return true;
}
class RollingByteWindow {
private readonly bytes: Buffer;
private start = 0;
private length = 0;
constructor(private readonly capacity: number) {
this.bytes = Buffer.allocUnsafe(capacity);
}
append(source: Buffer, from = 0, to = source.byteLength): void {
const sourceLength = to - from;
if (sourceLength <= 0) return;
if (sourceLength >= this.capacity) {
source.copy(this.bytes, 0, to - this.capacity, to);
this.start = 0;
this.length = this.capacity;
return;
}
const overflow = Math.max(0, this.length + sourceLength - this.capacity);
this.start = (this.start + overflow) % this.capacity;
this.length -= overflow;
const writeAt = (this.start + this.length) % this.capacity;
const firstLength = Math.min(sourceLength, this.capacity - writeAt);
source.copy(this.bytes, writeAt, from, from + firstLength);
if (firstLength < sourceLength) {
source.copy(this.bytes, 0, from + firstLength, to);
}
this.length += sourceLength;
}
appendByte(byte: number): void {
if (this.length < this.capacity) {
this.bytes[(this.start + this.length) % this.capacity] = byte;
this.length += 1;
return;
}
this.bytes[this.start] = byte;
this.start = (this.start + 1) % this.capacity;
}
appendWindow(source: RollingByteWindow): void {
if (source.length === 0) return;
const firstLength = Math.min(source.length, source.capacity - source.start);
this.append(source.bytes, source.start, source.start + firstLength);
if (firstLength < source.length) {
this.append(source.bytes, 0, source.length - firstLength);
}
}
reset(): void {
this.start = 0;
this.length = 0;
}
digest(): string {
const hash = createHash("sha256");
if (this.length === 0) return hash.digest("hex");
const firstLength = Math.min(this.length, this.capacity - this.start);
hash.update(this.bytes.subarray(this.start, this.start + firstLength));
if (firstLength < this.length) {
hash.update(this.bytes.subarray(0, this.length - firstLength));
}
return hash.digest("hex");
}
}
function rebuildRequired(reason: UsageLedgerRebuildReason): UsageLedgerRebuildRequiredError {
return new UsageLedgerRebuildRequiredError(reason);
}
function captureRangeIntoWindow(
fd: number,
from: number,
to: number,
scratch: Buffer,
window: RollingByteWindow,
signal: AbortSignal | undefined,
): void {
for (let position = from; position < to;) {
throwIfAborted(signal);
const requested = Math.min(scratch.byteLength, to - position);
const read = readSync(fd, scratch, 0, requested, position);
if (read === 0) throw rebuildRequired("shrink");
window.append(scratch, 0, read);
position += read;
}
}
async function digestRangeCooperatively(
fd: number,
from: number,
to: number,
buffer: Buffer,
signal: AbortSignal | undefined,
): Promise<string> {
const hash = createHash("sha256");
for (let position = from; position < to;) {
throwIfAborted(signal);
const requested = Math.min(buffer.byteLength, to - position);
const read = readSync(fd, buffer, 0, requested, position);
if (read === 0) throw rebuildRequired("shrink");
hash.update(buffer.subarray(0, read));
position += read;
if (position < to) await new Promise<void>(resolve => setTimeout(resolve, 0));
}
return hash.digest("hex");
}
/** Synchronous digest of a short range; used to re-validate a 64 KiB boundary. */
function digestRangeSync(fd: number, from: number, to: number, buffer: Buffer): string {
const hash = createHash("sha256");
for (let position = from; position < to;) {
const requested = Math.min(buffer.byteLength, to - position);
const read = readSync(fd, buffer, 0, requested, position);
if (read === 0) throw rebuildRequired("shrink");
hash.update(buffer.subarray(0, read));
position += read;
}
return hash.digest("hex");
}
/**
* Cooperatively scans a full ledger or validated append suffix with bounded memory.
*
* The opened EOF is the snapshot boundary: bytes appended after the initial fstat are
* deliberately left for the next scan. Rows are framed as raw bytes before UTF-8 decoding,
* so a multi-byte character may safely cross any read boundary. Only LF-terminated rows are
* published; a torn final write is skipped rather than accepted prematurely.
*/
export async function scanUsageLedgerCooperatively(
options: ScanUsageLedgerOptions,
): Promise<UsageLedgerScanResult> {
const chunkBytes = options.chunkBytes ?? USAGE_LEDGER_READ_CHUNK_BYTES;
if (!Number.isSafeInteger(chunkBytes) || chunkBytes <= 0 || chunkBytes > USAGE_LEDGER_READ_CHUNK_BYTES) {
throw new RangeError(`usage ledger chunk bytes must be between 1 and ${USAGE_LEDGER_READ_CHUNK_BYTES}`);
}
const startAtBytes = options.startAtBytes ?? 0;
if (!Number.isSafeInteger(startAtBytes) || startAtBytes < 0) {
throw new RangeError("usage ledger start offset must be a non-negative safe integer");
}
if (startAtBytes > 0
&& (options.expectedIdentityKey === undefined
|| options.expectedProcessedThroughDigest === undefined)) {
throw new TypeError(
"usage ledger append scan requires expectedIdentityKey and expectedProcessedThroughDigest",
);
}
throwIfAborted(options.signal);
const path = usageLogPath();
let fd: number;
try {
fd = openSync(path, "r");
} catch (error) {
if (isMissingFileError(error)) {
if (startAtBytes > 0
|| (options.expectedIdentityKey
&& options.expectedIdentityKey !== usageLogIdentityKey(null))) {
throw rebuildRequired("identity_mismatch");
}
return {
revision: null,
parsedRows: 0,
invalidRows: 0,
oversizedRows: 0,
bytesRead: 0,
processedThroughBytes: 0,
processedThroughDigest: createHash("sha256").digest("hex"),
};
}
throw error;
}
const decoder = new TextDecoder("utf-8", { fatal: true });
const chunk = Buffer.allocUnsafe(chunkBytes);
const line = Buffer.allocUnsafe(USAGE_LEDGER_MAX_LINE_BYTES);
let lineLength = 0;
let droppingOversizedLine = false;
let parsedRows = 0;
let invalidRows = 0;
let oversizedRows = 0;
let bytesRead = 0;
let bytesSinceYield = 0;
let linesSinceYield = 0;
let lastYieldAt = performance.now();
let processedThroughBytes = startAtBytes;
const capturedHash = createHash("sha256");
const committedTail = new RollingByteWindow(USAGE_LEDGER_BOUNDARY_DIGEST_BYTES);
const pendingTail = new RollingByteWindow(USAGE_LEDGER_BOUNDARY_DIGEST_BYTES);
let pendingLineBytes = 0;
try {
const openedRevision = revisionFromStat(path, fd, fstatSync(fd));
const scanEnd = openedRevision.size;
const openedIdentityKey = usageLogIdentityKey(openedRevision);
if (options.expectedIdentityKey && options.expectedIdentityKey !== openedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (scanEnd < startAtBytes) throw rebuildRequired("shrink");
if (startAtBytes > 0) {
const preceding = Buffer.allocUnsafe(1);
const read = readSync(fd, preceding, 0, 1, startAtBytes - 1);
if (read !== 1) throw rebuildRequired("shrink");
if (preceding[0] !== 0x0a) throw rebuildRequired("boundary_mismatch");
}
if (options.expectedProcessedThroughDigest !== undefined) {
captureRangeIntoWindow(
fd,
Math.max(0, startAtBytes - USAGE_LEDGER_BOUNDARY_DIGEST_BYTES),
startAtBytes,
line,
committedTail,
options.signal,
);
if (committedTail.digest() !== options.expectedProcessedThroughDigest) {
throw rebuildRequired("content_changed");
}
}
const publishLine = (rowStartBytes: number): void => {
if (lineLength === 0 || isJsonWhitespace(line, lineLength)) return;
let entry: PersistedUsageEntry | undefined;
try {
const text = decoder.decode(line.subarray(0, lineLength));
entry = normalizePersistedUsageRow(JSON.parse(text));
} catch {
invalidRows += 1;
return;
}
if (!entry) {
invalidRows += 1;
return;
}
options.onEntry(entry, rowStartBytes);
parsedRows += 1;
};
for (let position = startAtBytes; position < scanEnd;) {
throwIfAborted(options.signal);
const chunkStart = position;
const requested = Math.min(chunk.byteLength, scanEnd - position);
const read = readSync(fd, chunk, 0, requested, position);
if (read === 0) throw rebuildRequired("shrink");
position += read;
bytesRead += read;
bytesSinceYield += read;
capturedHash.update(chunk.subarray(0, read));
let cursor = 0;
while (cursor < read) {
const newline = chunk.indexOf(0x0a, cursor);
const segmentEnd = newline >= 0 && newline < read ? newline : read;
const segmentLength = segmentEnd - cursor;
if (segmentLength > 0) {
pendingTail.append(chunk, cursor, segmentEnd);
pendingLineBytes = Math.min(
USAGE_LEDGER_BOUNDARY_DIGEST_BYTES,
pendingLineBytes + segmentLength,
);
}
if (!droppingOversizedLine) {
if (lineLength + segmentLength > USAGE_LEDGER_MAX_LINE_BYTES) {
oversizedRows += 1;
droppingOversizedLine = true;
lineLength = 0;
} else if (segmentLength > 0) {
chunk.copy(line, lineLength, cursor, segmentEnd);
lineLength += segmentLength;
}
}
if (newline < 0 || newline >= read) break;
linesSinceYield += 1;
const rowStartBytes = processedThroughBytes;
processedThroughBytes = chunkStart + newline + 1;
if (pendingLineBytes + 1 >= USAGE_LEDGER_BOUNDARY_DIGEST_BYTES) {
committedTail.reset();
}
committedTail.appendWindow(pendingTail);
committedTail.appendByte(0x0a);
pendingTail.reset();
pendingLineBytes = 0;
if (droppingOversizedLine) {
droppingOversizedLine = false;
} else {
publishLine(rowStartBytes);
}
lineLength = 0;
cursor = newline + 1;
// Bytes do not bound normalization/aggregation time on a busy host. Let
// network callbacks, timers and cancellation run even within the final chunk.
// Only elapsed time triggers this yield: a row count here would yield several
// times per chunk of cheap rows and slow a cold scan (the chunk boundary below
// keeps the row limit). setTimeout, unlike setImmediate, lets due timers run.
if (performance.now() - lastYieldAt >= MAX_SCAN_SLICE_MS) {
await new Promise<void>(resolve => setTimeout(resolve, 0));
throwIfAborted(options.signal);
lastYieldAt = performance.now();
bytesSinceYield = 0;
linesSinceYield = 0;
}
}
if (position < scanEnd
&& (bytesSinceYield >= USAGE_LEDGER_READ_CHUNK_BYTES || linesSinceYield >= 1_000)) {
await new Promise<void>(resolve => setTimeout(resolve, 0));
lastYieldAt = performance.now();
bytesSinceYield = 0;
linesSinceYield = 0;
}
}
// A non-empty suffix without LF is not a committed JSONL row, even when it happens
// to contain valid JSON. Count it as invalid and leave it out of the aggregate.
if (!droppingOversizedLine && lineLength > 0 && !isJsonWhitespace(line, lineLength)) {
invalidRows += 1;
}
throwIfAborted(options.signal);
const endingRevision = revisionFromStat(path, fd, fstatSync(fd));
if (usageLogIdentityKey(endingRevision) !== openedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (endingRevision.size < scanEnd) throw rebuildRequired("shrink");
const pathRevision = currentUsageLogRevision();
if (!pathRevision || usageLogIdentityKey(pathRevision) !== openedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (pathRevision.size < scanEnd) throw rebuildRequired("shrink");
// A pure append changes size/mtime/ctime but leaves the captured prefix intact and
// is safe to ignore until the next scan. Re-read only that prefix when a mutation
// was observed, so a same-inode rewrite (including rewrite + growth) cannot publish
// a mixture of old and new rows after a cooperative yield.
const mutationObserved = usageLogRevisionKey(endingRevision) !== usageLogRevisionKey(openedRevision)
|| usageLogRevisionKey(pathRevision) !== usageLogRevisionKey(openedRevision);
if (mutationObserved) {
const capturedDigest = capturedHash.digest("hex");
const verifiedDigest = await digestRangeCooperatively(
fd,
startAtBytes,
scanEnd,
line,
options.signal,
);
const verifiedFdRevision = revisionFromStat(path, fd, fstatSync(fd));
const verifiedPathRevision = currentUsageLogRevision();
if (capturedDigest !== verifiedDigest) throw rebuildRequired("content_changed");
if (!verifiedPathRevision
|| usageLogIdentityKey(verifiedFdRevision) !== openedIdentityKey
|| usageLogIdentityKey(verifiedPathRevision) !== openedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (verifiedFdRevision.size < scanEnd || verifiedPathRevision.size < scanEnd) {
throw rebuildRequired("shrink");
}
}
throwIfAborted(options.signal);
const processedThroughDigest = committedTail.digest();
return {
revision: openedRevision,
parsedRows,
invalidRows,
oversizedRows,
bytesRead,
processedThroughBytes,
processedThroughDigest,
};
} finally {
closeSync(fd);
}
}
export interface ReadUsageLedgerRowsOptions {
/** Ascending, distinct row-start offsets reported by an earlier scan of the same file. */
offsets: readonly number[];
/**
* Folds rows as they are decoded instead of retaining them in `entries`. Callback work shares
* the cooperative time slice. Results are provisional until this read resolves: discard them
* if identity or content validation fails, as with the full-ledger scanner's callback.
*/
onEntry?: (entry: PersistedUsageEntry, rowStartBytes: number) => void;
/** Identity those offsets were captured under; a replaced file invalidates all of them. */
expectedIdentityKey: string;
/** Revision the offsets were captured under. Any movement re-validates the boundary. */
expectedRevisionKey?: string;
/** Snapshot boundary the offsets belong to; every offset must fall before it. */
expectedProcessedThroughBytes?: number;
/** Trailing digest at that boundary, verified when the revision moved. */
expectedProcessedThroughDigest?: string;
/**
* Physical file size when the snapshot was published. A torn suffix makes this larger than
* the processed boundary, so growth is measured against it rather than the boundary.
*/
expectedFileSize?: number;
signal?: AbortSignal;
}
export interface UsageLedgerRowRead {
/** Empty when an onEntry callback consumes the rows. */
entries: PersistedUsageEntry[];
/** Offsets that no longer frame a complete, normalizable row. */
unreadableOffsets: number[];
}
/**
* Re-reads only the rows a derived index selected, addressed by their recorded byte offsets.
*
* This is the read half of the offset index: the caller already knows which rows match, so a
* filtered query pays to decode and normalize its candidates instead of the whole ledger. Every
* bound still belongs to this scanner -- the 1 MiB row ceiling, the opened-file identity, and the
* refusal to publish a torn final row -- because an offset reader with its own limits would be a
* second storage policy.
*
* A row that no longer parses at its recorded offset means the index and the ledger disagree;
* the caller is expected to drop the index and rebuild rather than serve a partial answer.
*/
export async function readUsageLedgerRowsAtOffsets(
options: ReadUsageLedgerRowsOptions,
): Promise<UsageLedgerRowRead> {
throwIfAborted(options.signal);
const path = usageLogPath();
let fd: number;
try {
fd = openSync(path, "r");
} catch (error) {
if (isMissingFileError(error)) {
// An empty selection against a missing ledger is an empty installation. A non-empty
// selection against one means the projection outlived its source.
if (options.offsets.length === 0
&& options.expectedIdentityKey === usageLogIdentityKey(null)) {
return { entries: [], unreadableOffsets: [] };
}
throw rebuildRequired("identity_mismatch");
}
throw error;
}
const decoder = new TextDecoder("utf-8", { fatal: true });
const window = Buffer.allocUnsafe(USAGE_LEDGER_ROW_READ_WINDOW_BYTES);
// One byte past the ceiling: a row of exactly USAGE_LEDGER_MAX_LINE_BYTES is accepted by the
// scanner, so its terminating LF has to fit too.
const row = Buffer.allocUnsafe(USAGE_LEDGER_MAX_LINE_BYTES + 1);
let windowStart = -1;
let windowLength = 0;
try {
const openedRevision = revisionFromStat(path, fd, fstatSync(fd));
if (usageLogIdentityKey(openedRevision) !== options.expectedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
const size = openedRevision.size;
const boundary = options.expectedProcessedThroughBytes;
if (boundary !== undefined) {
if (size < boundary) throw rebuildRequired("shrink");
for (const offset of options.offsets) {
// An offset at or past the boundary belongs to a different snapshot than the one the
// caller validated.
if (offset < 0 || offset >= boundary) throw rebuildRequired("content_changed");
}
}
// The revision moved since the index snapshot: verify the boundary the offsets were
// captured at before trusting any of them. A same-size rewrite of that tail is caught
// here; an append past the boundary leaves it unchanged.
const revisionMoved = options.expectedRevisionKey !== undefined
&& usageLogRevisionKey(openedRevision) !== options.expectedRevisionKey;
const physical = options.expectedFileSize ?? boundary;
if (revisionMoved) {
if (boundary === undefined || options.expectedProcessedThroughDigest === undefined) {
throw rebuildRequired("content_changed");
}
// Growth is measured against the snapshot's physical size, not the processed boundary: a
// torn suffix leaves the file larger than the boundary without being new content. A file
// that did not grow cannot be an append, so a moved revision is an in-place rewrite and
// the trailing digest cannot see one earlier than its 64 KiB window.
if (physical === undefined || size <= physical) throw rebuildRequired("content_changed");
const digest = digestRangeSync(
fd,
Math.max(0, boundary - USAGE_LEDGER_BOUNDARY_DIGEST_BYTES),
boundary,
row,
);
if (digest !== options.expectedProcessedThroughDigest) {
throw rebuildRequired("content_changed");
}
}
const lineAt = (offset: number): Buffer | null => {
if (offset >= windowStart && offset < windowStart + windowLength) {
const from = offset - windowStart;
// Search only the bytes actually read: the rest of the buffer is uninitialized.
const newline = window.subarray(from, windowLength).indexOf(0x0a);
if (newline >= 0) return window.subarray(from, from + newline);
// The cached window ends without a newline; a torn suffix has no complete row.
if (windowStart + windowLength >= size) return null;
}
const requested = Math.min(window.byteLength, size - offset);
if (requested <= 0) return null;
const read = readSync(fd, window, 0, requested, offset);
if (read <= 0) return null;
windowStart = offset;
windowLength = read;
const newline = window.subarray(0, read).indexOf(0x0a);
if (newline >= 0) return window.subarray(0, newline);
// The row spans more than one window: copy the prefix and grow up to the row ceiling.
window.copy(row, 0, 0, read);
let length = read;
while (length <= USAGE_LEDGER_MAX_LINE_BYTES) {
const want = Math.min(row.byteLength - length, size - offset - length);
if (want <= 0) return null;
const more = readSync(fd, row, length, want, offset + length);
if (more <= 0) return null;
const found = row.subarray(length, length + more).indexOf(0x0a);
if (found >= 0) {
windowStart = -1;
windowLength = 0;
return row.subarray(0, length + found);
}
length += more;
}
return null;
};
const walk = async (visit: (bytes: Buffer, offset: number) => boolean): Promise<number[]> => {
const unreadableOffsets: number[] = [];
let lastYieldAt = performance.now();
for (const offset of options.offsets) {
throwIfAborted(options.signal);
if (!Number.isSafeInteger(offset) || offset < 0 || offset >= size) {
unreadableOffsets.push(offset);
} else {
const bytes = lineAt(offset);
if (!bytes || !visit(bytes, offset)) unreadableOffsets.push(offset);
}
// A bounded candidate count does not bound decoding or aggregation time. Include the
// visitor in the same slice as the full scanner, and let timers/cancellation run even
// within one page. A single row remains atomic.
if (performance.now() - lastYieldAt >= MAX_SCAN_SLICE_MS) {
await new Promise<void>(resolve => setTimeout(resolve, 0));
throwIfAborted(options.signal);
lastYieldAt = performance.now();
}
}
return unreadableOffsets;
};
const entries: PersistedUsageEntry[] = [];
const captured = createHash("sha256");
const unreadableOffsets = await walk((bytes, offset) => {
captured.update(bytes);
captured.update("\0");
captured.update(String(offset));
let entry: PersistedUsageEntry | undefined;
try {
entry = normalizePersistedUsageRow(JSON.parse(decoder.decode(bytes)));
} catch {
entry = undefined;
}
// The index only holds rows the scanner normalized. A candidate that no longer parses or
// normalizes means the ledger moved under the projection, so it is unreadable rather than
// silently dropped: the caller rebuilds instead of reporting a smaller complete result.
if (!entry) return false;
if (options.onEntry) options.onEntry(entry, offset);
else entries.push(entry);
return true;
});
const capturedDigest = captured.digest("hex");
throwIfAborted(options.signal);
const endingRevision = revisionFromStat(path, fd, fstatSync(fd));
if (usageLogIdentityKey(endingRevision) !== options.expectedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (endingRevision.size < size) throw rebuildRequired("shrink");
const pathRevision = currentUsageLogRevision();
if (!pathRevision || usageLogIdentityKey(pathRevision) !== options.expectedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (pathRevision.size < size) throw rebuildRequired("shrink");
// Any write during the read can be an in-place rewrite, not just an append. Re-read the
// same offsets and compare, so a rewrite that lands between two batches cannot publish a
// mixture of old and new rows. A pure append leaves every selected row unchanged.
const mutated = usageLogRevisionKey(endingRevision) !== usageLogRevisionKey(openedRevision)
|| usageLogRevisionKey(pathRevision) !== usageLogRevisionKey(openedRevision);
if (mutated) {
// A write that did not grow the file is an in-place rewrite. Re-reading the selected rows
// cannot see a rewrite that turns a non-candidate into a match, so this must rebuild.
if (physical !== undefined && endingRevision.size <= physical) {
throw rebuildRequired("content_changed");
}
// Verification must read the file again, not re-hash a small page still in the read cache.
windowStart = -1;
windowLength = 0;
const verified = createHash("sha256");
const recheck = await walk((bytes, offset) => {
verified.update(bytes);
verified.update("\0");
verified.update(String(offset));
return true;
});
if (recheck.length !== unreadableOffsets.length
|| verified.digest("hex") !== capturedDigest) {
throw rebuildRequired("content_changed");
}
const verifiedFdRevision = revisionFromStat(path, fd, fstatSync(fd));
const verifiedPathRevision = currentUsageLogRevision();
if (!verifiedPathRevision
|| usageLogIdentityKey(verifiedFdRevision) !== options.expectedIdentityKey
|| usageLogIdentityKey(verifiedPathRevision) !== options.expectedIdentityKey) {
throw rebuildRequired("identity_mismatch");
}
if (verifiedFdRevision.size < size || verifiedPathRevision.size < size) {
throw rebuildRequired("shrink");
}
// Verification yields too. A later write could change bytes already checked, so retry
// rather than publish a result whose verification raced another mutation.
if (usageLogRevisionKey(verifiedFdRevision) !== usageLogRevisionKey(endingRevision)
|| usageLogRevisionKey(verifiedPathRevision) !== usageLogRevisionKey(pathRevision)) {
throw rebuildRequired("content_changed");
}
}
throwIfAborted(options.signal);
return { entries, unreadableOffsets };
} finally {
closeSync(fd);
}
}