@tanstack/ai-persistence
Version:
Composable state persistence for TanStack AI messages, runs, interrupts, metadata, and locks.
3,276 lines • 103 kB
JavaScript
import { LogConflictError, LogTruncatedError, OwnershipLostError } from "../types.js";
import { beforeAll, describe, expect, it, vi } from "vitest";
//#region src/testkit/conformance.ts
/**
* Shared conformance suite for the `AIPersistence` store contract.
*
* Every backend runs this identical suite — the in-memory reference store and
* every adapter you write against your own database — so that schema drift or
* an implementation gap fails immediately. It exercises every method of every
* store the persistence exposes and is the authoritative compatibility gate for
* the store interfaces in `../types.ts`.
*
* Covers the five chat state stores (`messages`, `activities`, `runs`,
* `interrupts`, `metadata`), the three generation stores (`generationRuns`,
* `artifacts`, `blobs`), and the opt-in session stores (`log`,
* `sessions`). The session store cases run only
* when the store is present, so they need no `skip` entry. Locks are not part of
* this suite — they are a separate coordination concern (`LockStore` +
* `withLocks`), not a store.
*
* SKIPPING (declare or fail): a backend that deliberately omits a store must
* declare it in `options.skip`, and one that omits an OPTIONAL store method
* must declare it in `options.skipMethods`. Anything absent and not declared
* fails the suite loudly, and anything declared absent is reported by vitest as
* a SKIPPED case, never as a pass. Silent gaps are not allowed: a case that did
* not run must never be indistinguishable from one that did. A chat-only
* adapter with no activity store therefore passes
* `skip: ['activities', 'generationRuns', 'artifacts', 'blobs']`, and a
* generation-only one skips the five state stores. `listByParentRun` is
* the exception. Subagent support is opt-in, its cases skip on their own, and
* a `'runs.listByParentRun'` entry is accepted but has no effect.
*
* NOT COVERED HERE: the four durable-run fields on `RunRecord` (`sandboxKey`,
* `detachedSince`, `cancelRequested`, `driverEpoch`). They exist for durable
* sandboxed runs, a chat app never writes them, and requiring them here made every
* backend implement four columns and an omitted-vs-explicit-undefined rule it had no
* use for. They are proven by `runDurableRunFieldsConformance` from
* `@tanstack/ai-sandbox/testkit`, next to the takeover and reaper suites that consume
* them. A backend that never runs sandboxes can leave the columns out.
*
* RESERVED RUN-ID PREFIX: `rc-` belongs to the `listReclaimable` case, which
* filters the method's (not thread-scoped) result down to `rc-` ids before an
* exact-set assertion. A new case in `describe('runs')` must NOT seed a run id
* starting with `rc-`, or it silently changes that expected set.
*/
/**
* Unwrap a value the store contract says must be present. Fails the test with a
* readable message instead of a non-null assertion (banned in this package) or
* an early `return` that would pass silently.
*/
function required(value, what) {
if (value == null) throw new Error(`AIPersistence conformance: expected ${what} to exist`);
return value;
}
/** Read a `BlobObject.body` to completion as one contiguous buffer. */
async function drainStream(stream) {
const chunks = [];
let total = 0;
const reader = stream.getReader();
for (;;) {
const { value, done } = await reader.read();
if (done) break;
chunks.push(value);
total += value.byteLength;
}
const bytes = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.byteLength;
}
return bytes;
}
/**
* Register a Vitest suite that validates `makePersistence()` against the full
* `AIPersistence` contract — every store it provides, and none it declares
* skipped.
*/
function runPersistenceConformance(name, makePersistence, options) {
const skip = new Set(options?.skip ?? []);
const skipMethods = new Set(options?.skipMethods ?? []);
const checks = new Set(options?.checks ?? []);
describe(`AIPersistence conformance: ${name}`, () => {
let persistence;
beforeAll(async () => {
persistence = await makePersistence();
});
/**
* Return the store for `key`, or `null` when the backend intentionally
* skips it. Throws (failing the test) when a store is missing but was not
* declared in `options.skip`.
*/
function resolveStore(key) {
const store = persistence.stores[key];
if (store) return store;
if (skip.has(key)) return null;
throw new Error(`AIPersistence conformance: store '${key}' is missing. Provide it, or pass { skip: ['${key}'] } if the omission is intentional.`);
}
/**
* Return the second store set from `options.twin`. Throws (failing the
* test) when the option is missing, mirroring `resolveStore`: a case
* that cannot run must never be reported as a pass.
*/
async function resolveTwin() {
const twin = options?.twin;
if (twin === void 0) throw new Error("AIPersistence conformance: Provide `twin`: a second store set on the same backend.");
return twin(persistence);
}
/**
* Narrow `runs` to a store that definitely implements the optional method
* `methodName`, so the case can call it without a non-null assertion.
*
* Returns `false` only when the omission was declared in
* `options.skipMethods` (the caller then reports a skip). An undeclared
* omission throws, mirroring `resolveStore`: a case that cannot run must
* never be reported as a pass.
*/
function hasRunsMethod(runs, methodName) {
if (runs[methodName]) return true;
const key = `runs.${methodName}`;
if (skipMethods.has(key)) return false;
throw new Error(`AIPersistence conformance: optional method '${key}' is not implemented. Implement it, or pass { skipMethods: ['${key}'] } if the omission is intentional.`);
}
function supportsSubagents(runs) {
return typeof runs.listByParentRun === "function";
}
describe("messages", () => {
it("round-trips a thread and returns [] for unknown threads", async (ctx) => {
const store = resolveStore("messages");
if (!store) return ctx.skip("store not provided");
expect(await store.loadThread("thread-unknown")).toEqual([]);
await store.saveThread("thread-msg", [{
role: "user",
content: "hi"
}, {
role: "assistant",
content: "hello"
}]);
const loaded = await store.loadThread("thread-msg");
expect(Array.isArray(loaded)).toBe(true);
expect(loaded).toEqual([{
role: "user",
content: "hi"
}, {
role: "assistant",
content: "hello"
}]);
await store.saveThread("thread-msg", [{
role: "user",
content: "redo"
}]);
expect(await store.loadThread("thread-msg")).toEqual([{
role: "user",
content: "redo"
}]);
});
it("round-trips rich message shapes with deep equality", async (ctx) => {
const store = resolveStore("messages");
if (!store) return ctx.skip("store not provided");
const rich = [
{
role: "user",
content: "plain string"
},
{
role: "assistant",
content: "",
toolCalls: [{
id: "call-1",
type: "function",
function: {
name: "search",
arguments: "{\"query\":\"weather in Paris\"}"
}
}]
},
{
role: "tool",
content: "{\"temperature\":21,\"unit\":\"C\"}",
toolCallId: "call-1"
},
{
role: "user",
content: [{
type: "text",
content: "What is in this image?"
}, {
type: "image",
source: {
type: "url",
value: "https://example.com/cat.png",
mimeType: "image/png"
}
}]
},
{
role: "assistant",
content: "Here is my answer.",
thinking: [{
content: "The user is asking about the image.",
signature: "sig-1"
}]
}
];
await store.saveThread("thread-rich", rich);
expect(await store.loadThread("thread-rich")).toEqual(rich);
});
it("round-trips message metadata", async (ctx) => {
if (!checks.has("messages.metadata")) return ctx.skip("opt-in check: pass { checks: ['messages.metadata'] }");
const store = resolveStore("messages");
if (!store) return ctx.skip("store not provided");
const withMetadata = [{
role: "user",
content: "hi",
metadata: { author: { id: "user-42" } }
}, {
role: "assistant",
content: "hello",
metadata: {
tanstack: { run: { id: "run-1" } },
custom: 1
}
}];
await store.saveThread("thread-metadata", withMetadata);
expect(await store.loadThread("thread-metadata")).toEqual(withMetadata);
});
});
describe("activities", () => {
it("round-trips a thread and returns [] for unknown threads", async (ctx) => {
const store = resolveStore("activities");
if (!store) return ctx.skip("store not provided");
expect(await store.loadActivities("thread-unknown")).toEqual([]);
await store.saveActivities("thread-act", [{
id: "act-1",
activityType: "SEARCH",
content: { query: "tanstack" },
index: 1
}]);
expect(await store.loadActivities("thread-act")).toEqual([{
id: "act-1",
activityType: "SEARCH",
content: { query: "tanstack" },
index: 1
}]);
await store.saveActivities("thread-act", [{
id: "act-2",
activityType: "PLAN",
content: { steps: [] },
index: 0
}]);
expect(await store.loadActivities("thread-act")).toEqual([{
id: "act-2",
activityType: "PLAN",
content: { steps: [] },
index: 0
}]);
});
});
describe("runs", () => {
it("creates, resumes idempotently, updates, and gets", async (ctx) => {
const store = resolveStore("runs");
if (!store) return ctx.skip("store not provided");
expect(await store.get("run-missing")).toBeNull();
const created = await store.createOrResume({
runId: "run-1",
threadId: "thread-1",
startedAt: 1e3
});
expect(created).toMatchObject({
runId: "run-1",
threadId: "thread-1",
status: "running",
startedAt: 1e3
});
const resumed = await store.createOrResume({
runId: "run-1",
threadId: "thread-different",
startedAt: 9999
});
expect(resumed).toMatchObject({
runId: "run-1",
threadId: "thread-1",
startedAt: 1e3
});
await store.update("run-1", {
status: "completed",
finishedAt: 2e3,
usage: {
promptTokens: 3,
completionTokens: 4,
totalTokens: 7
}
});
const done = await store.get("run-1");
expect(done).toMatchObject({
runId: "run-1",
status: "completed",
finishedAt: 2e3,
usage: {
promptTokens: 3,
completionTokens: 4,
totalTokens: 7
}
});
const resumedAfterUpdate = await store.createOrResume({
runId: "run-1",
threadId: "thread-different",
startedAt: 9999
});
expect(resumedAfterUpdate).toEqual(done);
await store.update("run-1", {
status: "failed",
error: {
message: "boom",
code: "provider_overloaded"
}
});
const failed = await store.get("run-1");
expect(failed?.status).toBe("failed");
expect(failed?.error).toEqual({
message: "boom",
code: "provider_overloaded"
});
await store.update("run-absent", { status: "completed" });
expect(await store.get("run-absent")).toBeNull();
});
it("createOrResume never resurrects a finished run", async (ctx) => {
const store = resolveStore("runs");
if (!store) return ctx.skip("store not provided");
await store.createOrResume({
runId: "nc-1",
threadId: "nc-t",
startedAt: 10
});
await store.update("nc-1", {
status: "completed",
finishedAt: 20
});
const resumed = await store.createOrResume({
runId: "nc-1",
threadId: "nc-t",
startedAt: 999,
status: "running"
});
expect(resumed).toMatchObject({
runId: "nc-1",
status: "completed",
startedAt: 10,
finishedAt: 20
});
expect(await store.get("nc-1")).toMatchObject({
status: "completed",
startedAt: 10,
finishedAt: 20
});
});
it("round-trips subagent link fields and ignores them on resume", async (ctx) => {
const store = resolveStore("runs");
if (!store) return ctx.skip("store not provided");
if (!supportsSubagents(store)) return ctx.skip("runs.listByParentRun not implemented");
const parent = await store.createOrResume({
runId: "lp-parent",
threadId: "lp-thread",
startedAt: 1
});
expect(parent.parentRunId).toBeUndefined();
expect(parent.subagentRunId).toBeUndefined();
expect(parent.name).toBeUndefined();
const child = await store.createOrResume({
runId: "lp-child",
threadId: "subagent:lp-child",
startedAt: 2,
parentRunId: "lp-parent",
subagentRunId: "lp-sub",
name: "researcher"
});
expect(child).toMatchObject({
runId: "lp-child",
threadId: "subagent:lp-child",
startedAt: 2,
parentRunId: "lp-parent",
subagentRunId: "lp-sub",
name: "researcher"
});
const resumed = await store.createOrResume({
runId: "lp-child",
threadId: "lp-other-thread",
startedAt: 99,
parentRunId: "lp-other-parent",
subagentRunId: "lp-other-sub",
name: "writer"
});
expect(resumed).toMatchObject({
runId: "lp-child",
threadId: "subagent:lp-child",
startedAt: 2,
parentRunId: "lp-parent",
subagentRunId: "lp-sub",
name: "researcher"
});
expect(await store.get("lp-child")).toMatchObject({
parentRunId: "lp-parent",
subagentRunId: "lp-sub",
name: "researcher"
});
});
it("findActiveRun returns the most recent running run for a thread", async (ctx) => {
const store = resolveStore("runs");
if (!store) return ctx.skip("store not provided");
const thread = "thread-active";
expect(await store.findActiveRun(thread)).toBeNull();
await store.createOrResume({
runId: "active-1",
threadId: thread,
startedAt: 1e3
});
await store.createOrResume({
runId: "active-2",
threadId: thread,
startedAt: 2e3
});
expect(await store.findActiveRun(thread)).toMatchObject({
runId: "active-2",
status: "running"
});
await store.createOrResume({
runId: "other-1",
threadId: "thread-other",
startedAt: 3e3
});
expect(await store.findActiveRun(thread)).toMatchObject({ runId: "active-2" });
await store.update("active-2", {
status: "completed",
finishedAt: 2500
});
expect(await store.findActiveRun(thread)).toMatchObject({
runId: "active-1",
status: "running"
});
await store.update("active-1", {
status: "completed",
finishedAt: 1500
});
expect(await store.findActiveRun(thread)).toBeNull();
});
it("lists runs by thread when supported", async (ctx) => {
const runs = resolveStore("runs");
if (!runs) return ctx.skip("store not provided");
if (!hasRunsMethod(runs, "listByThread")) return ctx.skip("runs.listByThread not implemented");
await runs.createOrResume({
runId: "lt-b",
threadId: "lt",
startedAt: 2
});
await runs.createOrResume({
runId: "lt-a",
threadId: "lt",
startedAt: 1
});
const listed = await runs.listByThread("lt");
expect(listed.map((r) => r.runId)).toEqual(["lt-a", "lt-b"]);
});
it("lists runs by thread with their current status and finishedAt", async (ctx) => {
if (!checks.has("runs.listByThread.state")) return ctx.skip("opt-in check: pass { checks: ['runs.listByThread.state'] }");
const runs = resolveStore("runs");
if (!runs) return ctx.skip("store not provided");
if (!hasRunsMethod(runs, "listByThread")) return ctx.skip("runs.listByThread not implemented");
await runs.createOrResume({
runId: "lts-a",
threadId: "lts",
startedAt: 1
});
await runs.update("lts-a", {
status: "completed",
finishedAt: 5
});
expect(await runs.listByThread("lts")).toEqual([expect.objectContaining({
runId: "lts-a",
status: "completed",
startedAt: 1,
finishedAt: 5
})]);
});
it("lists child runs by parent when supported", async (ctx) => {
const runs = resolveStore("runs");
if (!runs) return ctx.skip("store not provided");
if (!supportsSubagents(runs)) return ctx.skip("runs.listByParentRun not implemented");
await runs.createOrResume({
runId: "lbp-parent",
threadId: "lbp-thread",
startedAt: 1
});
await runs.createOrResume({
runId: "lbp-b",
threadId: "subagent:lbp-b",
startedAt: 20,
parentRunId: "lbp-parent",
subagentRunId: "lbp-sub-b",
name: "seo"
});
await runs.createOrResume({
runId: "lbp-a",
threadId: "subagent:lbp-a",
startedAt: 10,
parentRunId: "lbp-parent",
subagentRunId: "lbp-sub-a",
name: "researcher"
});
await runs.createOrResume({
runId: "lbp-other",
threadId: "subagent:lbp-other",
startedAt: 5,
parentRunId: "lbp-elsewhere",
subagentRunId: "lbp-sub-other",
name: "writer"
});
const listed = await runs.listByParentRun("lbp-parent");
expect(listed.map((run) => run.runId)).toEqual(["lbp-a", "lbp-b"]);
expect(listed.map((run) => run.name)).toEqual(["researcher", "seo"]);
expect(await runs.listByParentRun("lbp-missing")).toEqual([]);
});
it("lists reclaimable detached runs when supported", async (ctx) => {
const runs = resolveStore("runs");
if (!runs) return ctx.skip("store not provided");
if (!hasRunsMethod(runs, "listReclaimable")) return ctx.skip("runs.listReclaimable not implemented");
const now = 1e4;
const ttlMs = 5e3;
const cutoff = 5e3;
await runs.createOrResume({
runId: "rc-included",
threadId: "rc-t",
startedAt: 1
});
await runs.update("rc-included", { detachedSince: 1e3 });
await runs.createOrResume({
runId: "rc-boundary",
threadId: "rc-t",
startedAt: 1
});
await runs.update("rc-boundary", { detachedSince: cutoff });
await runs.createOrResume({
runId: "rc-too-recent",
threadId: "rc-t",
startedAt: 1
});
await runs.update("rc-too-recent", { detachedSince: 5001 });
await runs.createOrResume({
runId: "rc-completed",
threadId: "rc-t",
startedAt: 1
});
await runs.update("rc-completed", {
detachedSince: 1e3,
status: "completed",
finishedAt: 2e3
});
await runs.createOrResume({
runId: "rc-never-detached",
threadId: "rc-t",
startedAt: 1
});
const ourIds = (await runs.listReclaimable({
now,
ttlMs
})).map((r) => r.runId).filter((id) => id.startsWith("rc-")).sort();
expect(ourIds).toEqual(["rc-boundary", "rc-included"]);
const zeroTtlRunId = `rc-${crypto.randomUUID()}`;
await runs.createOrResume({
runId: zeroTtlRunId,
threadId: "rc-t",
startedAt: 1
});
await runs.update(zeroTtlRunId, { detachedSince: now });
const zeroTtlReclaimable = await runs.listReclaimable({
now,
ttlMs: 0
});
expect(zeroTtlReclaimable.find((r) => r.runId === zeroTtlRunId)?.detachedSince).toBe(now);
const reattachedRunId = `rc-${crypto.randomUUID()}`;
await runs.createOrResume({
runId: reattachedRunId,
threadId: "rc-t",
startedAt: 1
});
await runs.update(reattachedRunId, { detachedSince: 1e3 });
await runs.update(reattachedRunId, { detachedSince: void 0 });
const afterReattach = await runs.listReclaimable({
now,
ttlMs
});
expect(afterReattach.some((r) => r.runId === reattachedRunId)).toBe(false);
const terminalRunIds = await Promise.all([
"completed",
"failed",
"aborted"
].map(async (status) => {
const runId = `rc-${crypto.randomUUID()}`;
await runs.createOrResume({
runId,
threadId: "rc-t",
startedAt: 1
});
await runs.update(runId, {
status,
detachedSince: 1e3,
finishedAt: 2e3
});
return runId;
}));
const afterTerminal = await runs.listReclaimable({
now,
ttlMs
});
expect(afterTerminal.some((r) => terminalRunIds.includes(r.runId))).toBe(false);
const interruptedRunId = `rc-${crypto.randomUUID()}`;
await runs.createOrResume({
runId: interruptedRunId,
threadId: "rc-t",
startedAt: 1
});
await runs.update(interruptedRunId, {
status: "interrupted",
detachedSince: 1e3
});
const afterInterrupted = await runs.listReclaimable({
now,
ttlMs
});
expect(afterInterrupted.some((r) => r.runId === interruptedRunId)).toBe(false);
});
});
describe("interrupts", () => {
it("creates, resolves, cancels, and lists by thread and run", async (ctx) => {
const store = resolveStore("interrupts");
if (!store) return ctx.skip("store not provided");
expect(await store.get("int-missing")).toBeNull();
await store.create({
interruptId: "int-1",
runId: "run-i",
threadId: "thread-i",
requestedAt: 10,
payload: {
tool: "search",
args: { q: "x" }
}
});
await store.create({
interruptId: "int-2",
runId: "run-i",
threadId: "thread-i",
requestedAt: 20,
payload: { tool: "write" }
});
await store.create({
interruptId: "int-3",
runId: "run-other",
threadId: "thread-i",
requestedAt: 30,
payload: {}
});
const one = await store.get("int-1");
expect(one).toMatchObject({
interruptId: "int-1",
runId: "run-i",
threadId: "thread-i",
status: "pending",
requestedAt: 10,
payload: {
tool: "search",
args: { q: "x" }
}
});
expect((await store.list("thread-i")).map((r) => r.interruptId)).toEqual([
"int-1",
"int-2",
"int-3"
]);
expect((await store.listByRun("run-i")).map((r) => r.interruptId)).toEqual(["int-1", "int-2"]);
expect((await store.listPending("thread-i")).map((r) => r.interruptId)).toEqual([
"int-1",
"int-2",
"int-3"
]);
await store.resolve("int-1", { ok: true });
const resolved = await store.get("int-1");
expect(resolved?.status).toBe("resolved");
expect(resolved?.response).toEqual({ ok: true });
expect(typeof resolved?.resolvedAt).toBe("number");
await store.cancel("int-2");
const cancelled = await store.get("int-2");
expect(cancelled?.status).toBe("cancelled");
expect(typeof cancelled?.resolvedAt).toBe("number");
expect((await store.listPending("thread-i")).map((r) => r.interruptId)).toEqual(["int-3"]);
expect((await store.listPendingByRun("run-i")).map((r) => r.interruptId)).toEqual([]);
});
it("create is insert-if-absent: a duplicate id never clobbers a resolved interrupt", async (ctx) => {
const store = resolveStore("interrupts");
if (!store) return ctx.skip("store not provided");
await store.create({
interruptId: "int-dup",
runId: "run-dup",
threadId: "thread-dup",
requestedAt: 100,
payload: { attempt: 1 }
});
await store.resolve("int-dup", { answer: 42 });
await store.create({
interruptId: "int-dup",
runId: "run-dup",
threadId: "thread-dup",
requestedAt: 200,
payload: { attempt: 2 }
});
const after = await store.get("int-dup");
expect(after?.status).toBe("resolved");
expect(after?.response).toEqual({ answer: 42 });
expect(after?.payload).toEqual({ attempt: 1 });
expect(after?.requestedAt).toBe(100);
});
it("lists ordered by requestedAt ascending even when inserts are out of order", async (ctx) => {
const store = resolveStore("interrupts");
if (!store) return ctx.skip("store not provided");
await store.create({
interruptId: "int-late",
runId: "run-order",
threadId: "thread-order",
requestedAt: 300,
payload: {}
});
await store.create({
interruptId: "int-early",
runId: "run-order",
threadId: "thread-order",
requestedAt: 100,
payload: {}
});
await store.create({
interruptId: "int-mid",
runId: "run-order",
threadId: "thread-order",
requestedAt: 200,
payload: {}
});
expect((await store.list("thread-order")).map((r) => r.interruptId)).toEqual([
"int-early",
"int-mid",
"int-late"
]);
expect((await store.listPending("thread-order")).map((r) => r.interruptId)).toEqual([
"int-early",
"int-mid",
"int-late"
]);
expect((await store.listByRun("run-order")).map((r) => r.interruptId)).toEqual([
"int-early",
"int-mid",
"int-late"
]);
});
});
describe("generationRuns", () => {
it("creates, resumes idempotently, updates, and gets", async (ctx) => {
const store = resolveStore("generationRuns");
if (!store) return ctx.skip("store not provided");
expect(await store.get("gen-missing")).toBeNull();
const created = await store.createOrResume({
runId: "gen-1",
threadId: "gen-thread-1",
activity: "image",
provider: "openai",
model: "gpt-image-1",
startedAt: 1e3
});
expect(created).toMatchObject({
runId: "gen-1",
activity: "image",
provider: "openai",
model: "gpt-image-1",
status: "running",
startedAt: 1e3
});
expect(created.threadId).toBe("gen-thread-1");
const resumed = await store.createOrResume({
runId: "gen-1",
activity: "video",
provider: "google",
model: "veo-3",
startedAt: 9999,
threadId: "thread-late"
});
expect(resumed).toMatchObject({
runId: "gen-1",
activity: "image",
provider: "openai",
model: "gpt-image-1",
startedAt: 1e3
});
expect(resumed.threadId).toBe("gen-thread-1");
await store.update("gen-1", {
status: "completed",
finishedAt: 2e3,
result: { images: [{ url: "https://example.com/a.png" }] },
artifacts: [{
role: "output",
artifactId: "art-1",
runId: "gen-1",
threadId: "thread-gen",
name: "a.png",
mimeType: "image/png",
size: 3,
createdAt: "2024-01-01T00:00:00.000Z",
source: {
activity: "image",
path: "images.0",
provider: "openai",
model: "gpt-image-1"
}
}],
usage: {
promptTokens: 1,
completionTokens: 2,
totalTokens: 3
}
});
const done = await store.get("gen-1");
expect(done).toMatchObject({
status: "completed",
finishedAt: 2e3,
result: { images: [{ url: "https://example.com/a.png" }] },
usage: {
promptTokens: 1,
completionTokens: 2,
totalTokens: 3
}
});
expect(done?.artifacts).toHaveLength(1);
expect(done?.artifacts?.[0]).toMatchObject({
artifactId: "art-1",
mimeType: "image/png"
});
await store.update("gen-1", {
status: "failed",
error: {
message: "boom",
code: "provider_error"
}
});
const failed = await store.get("gen-1");
expect(failed?.status).toBe("failed");
expect(failed?.error).toEqual({
message: "boom",
code: "provider_error"
});
await store.update("gen-absent", { status: "completed" });
expect(await store.get("gen-absent")).toBeNull();
});
it("findLatestForThread returns the most recently started linked run", async (ctx) => {
const store = resolveStore("generationRuns");
if (!store) return ctx.skip("store not provided");
const thread = "thread-gen-latest";
expect(await store.findLatestForThread(thread)).toBeNull();
await store.createOrResume({
runId: "gen-late",
activity: "image",
provider: "openai",
model: "gpt-image-1",
startedAt: 3e3,
threadId: thread
});
await store.createOrResume({
runId: "gen-early",
activity: "image",
provider: "openai",
model: "gpt-image-1",
startedAt: 1e3,
threadId: thread
});
expect(await store.findLatestForThread(thread)).toMatchObject({ runId: "gen-late" });
await store.createOrResume({
runId: "gen-other",
activity: "image",
provider: "openai",
model: "gpt-image-1",
startedAt: 9e3,
threadId: "thread-gen-other"
});
await store.update("gen-late", {
status: "completed",
finishedAt: 3500
});
expect(await store.findLatestForThread(thread)).toMatchObject({
runId: "gen-late",
status: "completed"
});
expect(await store.findLatestForThread("thread-unlinked")).toBeNull();
});
});
describe("artifacts", () => {
const artifact = (overrides) => ({
runId: "run-art",
threadId: "thread-art",
blobKey: "artifacts/run-art/default",
name: "image.png",
mimeType: "image/png",
size: 3,
createdAt: 100,
...overrides
});
it("saves as an upsert, gets, and lists by run", async (ctx) => {
const store = resolveStore("artifacts");
if (!store) return ctx.skip("store not provided");
expect(await store.get("art-missing")).toBeNull();
expect(await store.list("run-unknown")).toEqual([]);
await store.save(artifact({
artifactId: "𐀀",
blobKey: "artifacts/run-art/art-a",
createdAt: 100
}));
await store.save(artifact({
artifactId: "",
sourceUrl: "https://provider.example/expiring.png",
createdAt: 100
}));
await store.save(artifact({
artifactId: "art-c",
runId: "run-art-other"
}));
expect(await store.get("𐀀")).toMatchObject({
artifactId: "𐀀",
runId: "run-art",
threadId: "thread-art",
blobKey: "artifacts/run-art/art-a",
name: "image.png",
mimeType: "image/png",
size: 3,
createdAt: 100
});
expect(await store.get("")).toMatchObject({ sourceUrl: "https://provider.example/expiring.png" });
expect((await store.list("run-art")).map((r) => r.artifactId)).toEqual(["", "𐀀"]);
await store.save(artifact({
artifactId: "𐀀",
name: "renamed.png",
size: 9
}));
const updated = await store.get("𐀀");
expect(updated).toMatchObject({
name: "renamed.png",
size: 9
});
expect(updated?.blobKey).toBe("artifacts/run-art/default");
expect((await store.list("run-art")).map((r) => r.artifactId)).toEqual(["", "𐀀"]);
});
it("lists a thread in deterministic createdAt and artifactId order", async (ctx) => {
const store = resolveStore("artifacts");
if (!store) return ctx.skip("store not provided");
await store.save(artifact({
artifactId: "thread-b",
threadId: "thread-order",
createdAt: 2
}));
await store.save(artifact({
artifactId: "thread-a",
threadId: "thread-order",
createdAt: 2
}));
await store.save(artifact({
artifactId: "thread-early",
threadId: "thread-order",
createdAt: 1
}));
await store.save(artifact({
artifactId: "other",
threadId: "thread-other",
createdAt: 0
}));
expect((await store.listForThread("thread-order")).map((r) => r.artifactId)).toEqual([
"thread-early",
"thread-a",
"thread-b"
]);
});
it("orders artifact IDs by UTF-8 bytes after createdAt", async (ctx) => {
const store = resolveStore("artifacts");
if (!store) return ctx.skip("store not provided");
await store.save(artifact({
artifactId: "𐀀",
threadId: "thread-utf8",
createdAt: 1
}));
await store.save(artifact({
artifactId: "",
threadId: "thread-utf8",
createdAt: 1
}));
await store.save(artifact({
artifactId: "a",
threadId: "thread-utf8",
createdAt: 1
}));
expect((await store.listForThread("thread-utf8")).map((r) => r.artifactId)).toEqual([
"a",
"",
"𐀀"
]);
});
it("deletes one artifact and every artifact for a run", async (ctx) => {
const store = resolveStore("artifacts");
if (!store) return ctx.skip("store not provided");
await store.save(artifact({
artifactId: "art-d1",
runId: "run-del"
}));
await store.save(artifact({
artifactId: "art-d2",
runId: "run-del"
}));
await store.save(artifact({
artifactId: "art-keep",
runId: "run-keep"
}));
await store.delete("art-d1");
expect(await store.get("art-d1")).toBeNull();
expect((await store.list("run-del")).map((r) => r.artifactId)).toEqual(["art-d2"]);
await store.delete("art-d1");
await store.deleteForRun("run-del");
expect(await store.list("run-del")).toEqual([]);
expect(await store.get("art-d2")).toBeNull();
expect((await store.list("run-keep")).map((r) => r.artifactId)).toEqual(["art-keep"]);
await store.deleteForRun("run-del");
});
});
describe("blobs", () => {
it("round-trips bytes and metadata through put/get/head", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
expect(await store.get("blob-missing")).toBeNull();
expect(await store.head("blob-missing")).toBeNull();
const bytes = new Uint8Array([
1,
2,
3,
4
]);
const put = await store.put("blob/a", bytes, {
contentType: "image/png",
customMetadata: { runId: "run-blob" }
});
expect(put).toMatchObject({
key: "blob/a",
size: 4,
contentType: "image/png",
customMetadata: { runId: "run-blob" }
});
const object = required(await store.get("blob/a"), "blob/a");
expect(new Uint8Array(await object.arrayBuffer())).toEqual(bytes);
expect(object).toMatchObject({
key: "blob/a",
size: 4,
contentType: "image/png",
customMetadata: { runId: "run-blob" }
});
const head = await store.head("blob/a");
expect(head).toMatchObject({
key: "blob/a",
size: 4
});
await store.put("blob/text", "héllo");
const text = required(await store.get("blob/text"), "blob/text");
expect(await text.text()).toBe("héllo");
await store.put("blob/buffer", new Uint8Array([9, 9]).buffer);
const buffered = required(await store.get("blob/buffer"), "blob/buffer");
expect(new Uint8Array(await buffered.arrayBuffer())).toEqual(new Uint8Array([9, 9]));
});
it("accepts a stream body with no declared length", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
const bytes = (/* @__PURE__ */ new Uint8Array(65536)).map((_, i) => i % 251);
const lengthless = required(new Response(bytes).body, "response body").pipeThrough(new TransformStream());
const put = await store.put("blob/stream", lengthless, { contentType: "application/octet-stream" });
expect(put.size).toBe(bytes.byteLength);
const object = required(await store.get("blob/stream"), "blob/stream");
expect(new Uint8Array(await object.arrayBuffer())).toEqual(bytes);
expect(object.size).toBe(bytes.byteLength);
});
it("accepts expectedLength as an advisory hint for a stream body", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
const bytes = (/* @__PURE__ */ new Uint8Array(9216)).map((_, i) => i % 251);
const lengthless = required(new Response(bytes).body, "response body").pipeThrough(new TransformStream());
const put = await store.put("blob/stream-hint", lengthless, { expectedLength: bytes.byteLength });
expect(put.size).toBe(bytes.byteLength);
const object = required(await store.get("blob/stream-hint"), "blob/stream-hint");
expect(new Uint8Array(await object.arrayBuffer())).toEqual(bytes);
expect(object.size).toBe(bytes.byteLength);
});
it("serves a byte range and reports the slice it served", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
const bytes = (/* @__PURE__ */ new Uint8Array(1024)).map((_, i) => i % 251);
await store.put("blob/range", bytes);
const middle = required(await store.get("blob/range", { range: {
offset: 100,
length: 50
} }), "blob/range");
expect(new Uint8Array(await middle.arrayBuffer())).toEqual(bytes.slice(100, 150));
expect(middle.size).toBe(bytes.byteLength);
expect(middle.range).toEqual({
offset: 100,
length: 50
});
const tail = required(await store.get("blob/range", { range: { offset: 1e3 } }), "blob/range");
expect(new Uint8Array(await tail.arrayBuffer())).toEqual(bytes.slice(1e3));
expect(tail.range).toEqual({
offset: 1e3,
length: 24
});
const clamped = required(await store.get("blob/range", { range: {
offset: 1e3,
length: 1e3
} }), "blob/range");
expect(clamped.range).toEqual({
offset: 1e3,
length: 24
});
expect((await clamped.arrayBuffer()).byteLength).toBe(24);
const body = required(await store.get("blob/range", { range: {
offset: 10,
length: 5
} }), "blob/range").body;
if (body) expect(await drainStream(body)).toEqual(bytes.slice(10, 15));
const whole = required(await store.get("blob/range"), "blob/range");
expect(whole.range).toBeUndefined();
});
it("overwrites an existing key and deletes silently", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
const first = await store.put("blob/over", new Uint8Array([1]), {
contentType: "text/plain",
customMetadata: { v: "1" }
});
const second = await store.put("blob/over", new Uint8Array([
2,
2,
2
]), {
contentType: "application/octet-stream",
customMetadata: { v: "2" }
});
expect(second).toMatchObject({
size: 3,
contentType: "application/octet-stream",
customMetadata: { v: "2" }
});
if (first.etag !== void 0 && second.etag !== void 0) expect(second.etag).not.toBe(first.etag);
const after = required(await store.get("blob/over"), "blob/over");
expect(new Uint8Array(await after.arrayBuffer())).toEqual(new Uint8Array([
2,
2,
2
]));
await store.delete("blob/over");
expect(await store.get("blob/over")).toBeNull();
expect(await store.head("blob/over")).toBeNull();
await store.delete("blob/over");
});
it("lists by literal prefix in ascending key order", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
await store.put("list_/b", new Uint8Array([2]));
await store.put("list_/a", new Uint8Array([1]));
await store.put("list_/c", new Uint8Array([3]));
await store.put("list-x/d", new Uint8Array([4]));
await store.put("LIST_/e", new Uint8Array([5]));
const page = await store.list({ prefix: "list_/" });
expect(page.objects.map((o) => o.key)).toEqual([
"list_/a",
"list_/b",
"list_/c"
]);
expect(page.truncated).toBeFalsy();
expect((await store.list({ prefix: "list_/nothing-here" })).objects).toEqual([]);
});
it("pages with a cursor and returns an empty page for limit 0", async (ctx) => {
const store = resolveStore("blobs");
if (!store) return ctx.skip("store not provided");
for (const key of [
"page/a",
"page/b",
"page/c",
"page/d",
"page/e"
]) await store.put(key, new Uint8Array([1]));
const empty = await store.list({
prefix: "page/",
limit: 0
});
expect(empty.objects).toEqual([]);
expect(empty.truncated).toBeFalsy();
expect(empty.cursor).toBeUndefined();
const seen = [];
let cursor;
for (let guard = 0; guard < 10; guard++) {
const result = await store.list({
prefix: "page/",
limit: 2,
...cursor !== void 0 ? { cursor } : {}
});
seen.push(...result.objects.map((o) => o.key));
if (!result.truncated) break;
expect(result.cursor).toBe(result.objects.at(-1)?.key);
cursor = result.cursor;
}
expect(seen).toEqual([
"page/a",
"page/b",
"page/c",
"page/d",
"page/e"
]);
const exact = await store.list({
prefix: "page/",
limit: 5
});
expect(exact.objects.map((o) => o.key)).toEqual(seen);
expect(exact.truncated).toBeFalsy();
});
});
const newLog = () => `log-${crypto.randomUUID()}`;
const newHarness = () => `harness-${crypto.randomUUID()}`;
const countCalls = (store, logId) => {
let calls = 0;
return {
calls: () => calls,
stop: store.subscribe({
logId,
listener: () => {
calls += 1;
}
})
};
};
const waitForLateCalls = () => new Promise((resolve) => setTimeout(resolve, 50));
describe("log", () => {
const types = async (store, logId) => (await store.read({ logId })).map((entry) => [entry.seq, entry.record.type]);
it("writes a batch at consecutive positions and reads it in order", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{
type: "a",
n: 1
}, {
type: "b",
n: 2
}]
});
await store.append({
logId,
seq: 3,
epoch: 0,
records: [{
type: "c",
nested: { deep: [1] }
}]
});
expect(await store.read({ logId })).toEqual([
{
seq: 1,
record: {
type: "a",
n: 1
}
},
{
seq: 2,
record: {
type: "b",
n: 2
}
},
{
seq: 3,
record: {
type: "c",
nested: { deep: [1] }
}
}
]);
});
it("rejects a taken position or a gap with LogConflictError and writes nothing", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "first" }]
});
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "late-1" }, { type: "late-2" }]
})).rejects.toBeInstanceOf(LogConflictError);
await expect(store.append({
logId,
seq: 3,
epoch: 0,
records: [{ type: "gap" }]
})).rejects.toBeInstanceOf(LogConflictError);
expect(await types(store, logId)).toEqual([[1, "first"]]);
});
it("names the log in a LogConflictError", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "first" }]
});
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "late" }]
})).rejects.toMatchObject({
name: "LogConflictError",
logId,
seq: 1
});
});
it("rejects an epoch other than NO_OWNER_EPOCH on a log that no host acquired", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const write = store.append({
logId,
seq: 1,
epoch: 1,
records: [{ type: "fenced" }]
});
await expect(write).rejects.toBeInstanceOf(OwnershipLostError);
await expect(write).rejects.toMatchObject({
logId,
epoch: 1
});
expect(await store.read({ logId })).toEqual([]);
});
it("lets one of two concurrent appends at one position win", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const results = await Promise.allSettled([store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "left" }]
}), store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "right-1" }, { type: "right-2" }]
})]);
const won = results.filter((result) => result.status === "fulfilled");
const lost = results.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
expect(won).toHaveLength(1);
expect(lost).toHaveLength(1);
expect(lost[0]).toBeInstanceOf(LogConflictError);
const winner = results[0].status === "fulfilled" ? "left" : "right";
expect(await types(store, logId)).toEqual(winner === "left" ? [[1, "left"]] : [[1, "right-1"], [2, "right-2"]]);
});
it("writes nothing for an empty batch", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: []
});
expect(await store.read({ logId })).toEqual([]);
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "still-first" }]
});
expect(await types(store, logId)).toEqual([[1, "still-first"]]);
});
it("reads after a position, with a limit", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: [
{ type: "r1" },
{ type: "r2" },
{ type: "r3" },
{ type: "r4" }
]
});
const page = await store.read({
logId,
after: 1,
limit: 2
});
expect(page.map((entry) => entry.seq)).toEqual([2, 3]);
expect((await store.read({
logId,
after: 3
})).map((entry) => entry.seq)).toEqual([4]);
expect(await store.read({
logId,
after: 4
})).toEqual([]);
expect(await store.read({ logId: newLog() })).toEqual([]);
});
it("reads nothing with limit 0", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "r1" }, { type: "r2" }]
});
expect(await types(store, logId)).toHaveLength(2);
expect(await store.read({
logId,
limit: 0
})).toEqual([]);
});
it("keeps logs apart", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const left = newLog();
const right = newLog();
await store.append({
logId: left,
seq: 1,
epoch: 0,
records: [{ type: "left" }]
});
await store.append({
logId: right,
seq: 1,
epoch: 0,
records: [{ type: "right" }]
});
expect(await types(store, left)).toEqual([[1, "left"]]);
expect(await types(store, right)).toEqual([[1, "right"]]);
});
it("calls a listener after each append until it unsubscribes", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const counter = countCalls(store, logId);
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "one" }]
});
await expect.poll(counter.calls).toBe(1);
counter.stop();
await store.append({
logId,
seq: 2,
epoch: 0,
records: [{ type: "two" }]
});
await waitForLateCalls();
expect(counter.calls()).toBe(1);
});
it("does not share record objects with the caller", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const written = {
type: "shared",
list: [1]
};
await store.append({
logId,
seq: 1,
epoch: 0,
records: [written]
});
written.list.push(2);
const [first] = await store.read({ logId });
required(first, "the first log entry").record.type = "changed";
expect(await store.read({ logId })).toEqual([{
seq: 1,
record: {
type: "shared",
list: [1]
}
}]);
});
});
describe("log ownership", () => {
const LIVE_MS = 1e4;
const SHORT_MS = 400;
const waitPastShort = () => new Promise((resolve) => setTimeout(resolve, 500));
const claimOf = (logId, ownerId) => ({
logId,
harness: "h",
ownerId,
ttlMs: LIVE_MS
});
const handOver = async (store) => {
const logId = newLog();
await store.acquireOwnership({
...claimOf(logId, "a"),
ttlMs: SHORT_MS
});
await waitPastShort();
return {
logId,
taken: await store.acquireOwnership(claimOf(logId, "b"))
};
};
it("gives the first acquire epoch 1 and refuses a second owner", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
expect(await store.acquireOwnership(claimOf(logId, "a"))).toEqual({ epoch: 1 });
expect(await store.acquireOwnership(claimOf(logId, "b"))).toBeNull();
});
it("returns the same epoch when the owner acquires again", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
expect(await store.acquireOwnership(claimOf(logId, "a"))).toEqual({ epoch: 1 });
expect(await store.acquireOwnership(claimOf(logId, "a"))).toEqual({ epoch: 1 });
});
it("hands a log to a new owner after expiry with a higher epoch", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const { taken } = await handOver(store);
expect(taken).toEqual({
epoch: 2,
expired: true
});
});
it("fences a write from the previous owner", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const { logId } = await handOver(store);
await expect(store.append({
logId,
seq: 1,
epoch: 1,
records: [{ type: "old" }]
})).rejects.toBeInstanceOf(OwnershipLostError);
await store.append({
logId,
seq: 1,
epoch: 2,
records: [{ type: "new" }]
});
await expect(store.append({
logId,
seq: 1,
epoch: 1,
records: [{ type: "old" }]
})).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.read({ logId })).toEqual([{
seq: 1,
record: { type: "new" }
}]);
});
it("refuses NO_OWNER_EPOCH once a log was acquired", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.acquireOwnership(claimOf(logId, "a"));
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1
});
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "unowned" }]
})).rejects.toBeInstanceOf(OwnershipLostError);
await expect(store.append({
logId,
seq: 1,
epoch: 1,
records: [{ type: "released" }]
})).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.read({ logId })).toEqual([]);
});
it("renews only with the current epoch and owner", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const { logId } = await handOver(store);
expect(await store.renewOwnership({
logId,
ownerId: "a",
epoch: 1,
ttlMs: LIVE_MS
})).toBe(false);
expect(await store.renewOwnership({
logId,
ownerId: "a",
epoch: 2,
ttlMs: LIVE_MS
})).toBe(false);
expect(await store.renewOwnership({
logId,
ownerId: "b",
epoch: 2,
ttlMs: LIVE_MS
})).toBe(true);
});
it("renews after expiry when nobody took the log", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.acquireOwnership({
...claimOf(logId, "a"),
ttlMs: SHORT_MS
});
await waitPastShort();
expect(await store.renewOwnership({
logId,
ownerId: "a",
epoch: 1,
ttlMs: LIVE_MS
})).toBe(true);
expect(await store.acquireOwnership(claimOf(logId, "b"))).toBeNull();
});
it("releases and lets anyone acquire with the next epoch", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.acquireOwnership(claimOf(logId, "a"));
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1
});
expect(await store.renewOwnership({
logId,
ownerId: "a",
epoch: 1,
ttlMs: LIVE_MS
})).toBe(false);
expect(await store.acquireOwnership(claimOf(logId, "b"))).toEqual({ epoch: 2 });
});
it("ignores a release with a stale epoch", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const { logId } = await handOver(store);
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1
});
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 2
});
expect(await store.acquireOwnership(claimOf(logId, "c"))).toBeNull();
});
it("lists an expired log as orphaned", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const live = newLog();
const logId = newLog();
const harness = newHarness();
await store.acquireOwnership({
...claimOf(live, "a"),
harness
});
await store.acquireOwnership({
...claimOf(logId, "a"),
harness,
ttlMs: SHORT_MS
});
await waitPastShort();
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
})).toEqual([{
logId,
harness,
reason: "expired"
}]);
});
it("lists a released log with a due wake", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const later = newLog();
const logId = newLog();
const harness = newHarness();
for (const [id, wakeInMs] of [[later, LIVE_MS], [logId, SHORT_MS]]) {
await store.acquireOwnership({
...claimOf(id, "a"),
harness
});
await store.releaseOwnership({
logId: id,
ownerId: "a",
epoch: 1,
wakeInMs
});
}
await waitPastShort();
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
})).toEqual([{
logId,
harness,
reason: "wake"
}]);
});
it("clears the wake time on acquire", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const harness = newHarness();
await store.acquireOwnership({
...claimOf(logId, "a"),
harness
});
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1,
wakeInMs: 0
});
await store.acquireOwnership({
...claimOf(logId, "b"),
harness
});
await store.releaseOwnership({
logId,
ownerId: "b",
epoch: 2
});
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
})).toEqual([]);
});
it("filters orphaned logs by harness", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const first = newLog();
const second = newLog();
const h1 = newHarness();
const h2 = newHarness();
await store.acquireOwnership({
...claimOf(first, "a"),
harness: h1,
ttlMs: SHORT_MS
});
await store.acquireOwnership({
...claimOf(second, "a"),
harness: h2,
ttlMs: 410
});
await waitPastShort();
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [h1]
})).toEqual([{
logId: first,
harness: h1,
reason: "expired"
}]);
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [h2, h1]
})).toEqual([{
logId: first,
harness: h1,
reason: "expired"
}, {
logId: second,
harness: h2,
reason: "expired"
}]);
});
it("keeps a pinned log away from other affinities", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const harness = newHarness();
const claim = {
...claimOf(logId, "a"),
harness
};
await store.markHostAlive({
hostId: "host-1",
ttlMs: LIVE_MS
});
await store.acquireOwnership(claim);
await store.pinToWorkers({
logId,
epoch: 1,
affinity: "host-1"
});
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1,
wakeInMs: 0
});
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
})).toEqual([]);
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness],
affinity: "host-1"
})).toEqual([{
logId,
harness,
reason: "wake"
}]);
expect(await store.acquireOwnership({
...claim,
ownerId: "b",
affinity: "host-2"
})).toBeNull();
expect(await store.acquireOwnership({
...claim,
ownerId: "b",
affinity: "host-1"
})).toEqual({ epoch: 2 });
});
it("lets other hosts take a pinned log when the heartbeat of its host expired", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const harness = newHarness();
const pinnedLog = async (ttlMs) => {
const logId = newLog();
const host = `host-${logId}`;
await store.markHostAlive({
hostId: host,
ttlMs
});
await store.acquireOwnership({
...claimOf(logId, "a"),
harness,
affinity: host
});
await store.pinToWorkers({
logId,
epoch: 1,
affinity: host
});
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1,
wakeInMs: 0
});
return logId;
};
const kept = await pinnedLog(LIVE_MS);
const logId = await pinnedLog(SHORT_MS);
await waitPastShort();
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
})).toEqual([{
logId,
harness,
reason: "wake"
}]);
expect(await store.acquireOwnership({
...claimOf(kept, "b"),
harness
})).toBeNull();
expect(await store.acquireOwnership({
...claimOf(logId, "b"),
harness
})).toEqual({ epoch: 2 });
});
it("fences pinToWorkers", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.acquireOwnership(claimOf(logId, "a"));
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1
});
await store.acquireOwnership(claimOf(logId, "b"));
await expect(store.pinToWorkers({
logId,
epoch: 1,
affinity: "host-1"
})).rejects.toBeInstanceOf(OwnershipLostError);
await store.releaseOwnership({
logId,
ownerId: "b",
epoch: 2
});
expect(await store.acquireOwnership(claimOf(logId, "c"))).toEqual({ epoch: 3 });
const unowned = newLog();
await store.markHostAlive({
hostId: "host-1",
ttlMs: LIVE_MS
});
await expect(store.pinToWorkers({
logId: unowned,
epoch: 0,
affinity: "host-1"
})).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.acquireOwnership(claimOf(unowned, "a"))).toEqual({ epoch: 1 });
});
it("rejects a ttlMs that is not positive, or a negative wakeInMs", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await expect(store.acquireOwnership({
...claimOf(logId, "a"),
ttlMs: 0
})).rejects.toBeInstanceOf(RangeError);
expect(await store.acquireOwnership(claimOf(logId, "a"))).toEqual({ epoch: 1 });
await expect(store.renewOwnership({
logId,
ownerId: "a",
epoch: 1,
ttlMs: -1
})).rejects.toBeInstanceOf(RangeError);
await expect(store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1,
wakeInMs: -1
})).rejects.toBeInstanceOf(RangeError);
await expect(store.markHostAlive({
hostId: "host-1",
ttlMs: 0
})).rejects.toBeInstanceOf(RangeError);
expect(await store.acquireOwnership(claimOf(logId, "b"))).toBeNull();
});
it("lists nothing with limit 0", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.acquireOwnership({
...claimOf(logId, "a"),
harness: newHarness()
});
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1,
wakeInMs: 0
});
expect(await store.listLogsWithoutOwner({ limit: 1 })).toHaveLength(1);
expect(await store.listLogsWithoutOwner({ limit: 0 })).toEqual([]);
});
});
describe("log input lane", () => {
const inputOf = (text) => ({
type: "input",
text
});
const offerOf = (logId, inputId, text = "hi") => ({
logId,
harness: "h",
inputId,
record: inputOf(text)
});
const pendingIds = async (store, logId) => (await store.listOfferedInputs({ logId })).map((input) => input.inputId);
it("offers an input once and reports a duplicate with the same record", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
expect(await store.offerInput(offerOf(logId, "i1"))).toBe("offered");
expect(await store.offerInput(offerOf(logId, "i1"))).toBe("duplicate");
expect(await pendingIds(store, logId)).toEqual(["i1"]);
});
it("reports a conflict for the same id with another record", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.offerInput(offerOf(logId, "i1", "first"));
expect(await store.offerInput(offerOf(logId, "i1", "second"))).toBe("conflict");
expect(await store.listOfferedInputs({ logId })).toEqual([{
inputId: "i1",
record: inputOf("first"),
offeredAt: expect.any(Number)
}]);
});
it("keeps offers apart per log", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
expect(await store.offerInput(offerOf(newLog(), "i1"))).toBe("offered");
expect(await store.offerInput(offerOf(newLog(), "i1"))).toBe("offered");
});
it("lists pending offers in offer order", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.offerInput(offerOf(logId, "i2", "first"));
await store.offerInput(offerOf(logId, "i3", "second"));
await store.offerInput(offerOf(logId, "i1", "third"));
expect(await store.listOfferedInputs({ logId })).toEqual([
{
inputId: "i2",
record: inputOf("first"),
offeredAt: expect.any(Number)
},
{
inputId: "i3",
record: inputOf("second"),
offeredAt: expect.any(Number)
},
{
inputId: "i1",
record: inputOf("third"),
offeredAt: expect.any(Number)
}
]);
});
it("consumes offers in the same write as the records", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.offerInput(offerOf(logId, "i1", "one"));
await store.offerInput(offerOf(logId, "i2", "two"));
await store.append({
logId,
seq: 1,
epoch: 0,
records: [inputOf("one")],
consumes: ["i1"]
});
expect(await pendingIds(store, logId)).toEqual(["i2"]);
expect(await store.read({ logId })).toEqual([{
seq: 1,
record: inputOf("one")
}]);
expect(await store.offerInput(offerOf(logId, "i1", "one"))).toBe("duplicate");
await expect(store.append({
logId,
seq: 2,
epoch: 0,
records: [inputOf("one again")],
consumes: ["i1"]
})).rejects.toBeInstanceOf(TypeError);
expect(await store.read({
logId,
after: 1
})).toEqual([]);
});
it("rejects consumes of an id that is not pending and writes nothing", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.offerInput(offerOf(logId, "i1"));
await expect(store.append({
logId,
seq: 2,
epoch: 0,
records: [inputOf("hi")],
consumes: ["missing"]
})).rejects.toBeInstanceOf(LogConflictError);
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: [inputOf("hi")],
consumes: ["i1", "missing"]
})).rejects.toBeInstanceOf(TypeError);
expect(await store.read({ logId })).toEqual([]);
expect(await pendingIds(store, logId)).toEqual(["i1"]);
});
it("writes nothing when a consuming batch cannot become JSON", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.offerInput(offerOf(logId, "i1"));
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: [{
type: "input",
count: BigInt(1)
}],
consumes: ["i1"]
})).rejects.toThrow();
expect(await store.read({ logId })).toEqual([]);
expect(await pendingIds(store, logId)).toEqual(["i1"]);
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{
type: "input",
count: "1"
}],
consumes: ["i1"]
});
expect(await pendingIds(store, logId)).toEqual([]);
});
it("rejects an offer that cannot become JSON and stores nothing", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await expect(store.offerInput({
...offerOf(logId, "i1"),
record: {
type: "input",
count: BigInt(1)
}
})).rejects.toThrow();
expect(await store.listOfferedInputs({ logId })).toEqual([]);
expect(await store.offerInput(offerOf(logId, "i1"))).toBe("offered");
});
it("lists a log with pending offers and no live owner as orphaned", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const harness = newHarness();
const orphans = () => store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
});
await store.offerInput({
...offerOf(logId, "i1"),
harness
});
expect(await orphans()).toEqual([{
logId,
harness,
reason: "offered"
}]);
await store.acquireOwnership({
logId,
harness,
ownerId: "a",
ttlMs: 1e4
});
expect(await orphans()).toEqual([]);
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1,
wakeInMs: 0
});
expect(await orphans()).toEqual([{
logId,
harness,
reason: "offered"
}]);
const pinned = newLog();
const pinnedHarness = newHarness();
await store.markHostAlive({
hostId: "host-1",
ttlMs: 1e4
});
await store.acquireOwnership({
logId: pinned,
harness: pinnedHarness,
ownerId: "a",
ttlMs: 1e4
});
await store.pinToWorkers({
logId: pinned,
epoch: 1,
affinity: "host-1"
});
await store.releaseOwnership({
logId: pinned,
ownerId: "a",
epoch: 1
});
await store.offerInput({
...offerOf(pinned, "i1"),
harness: pinnedHarness
});
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [pinnedHarness]
})).toEqual([]);
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [pinnedHarness],
affinity: "host-1"
})).toEqual([{
logId: pinned,
harness: pinnedHarness,
reason: "offered"
}]);
});
it("keeps the harness of the owner or the first offer for a log", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const first = newHarness();
const other = newHarness();
const orphans = (harness) => store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
});
const offeredLog = newLog();
await store.offerInput({
...offerOf(offeredLog, "i1"),
harness: first
});
await store.offerInput({
...offerOf(offeredLog, "i2"),
harness: other
});
expect(await orphans(first)).toEqual([{
logId: offeredLog,
harness: first,
reason: "offered"
}]);
expect(await orphans(other)).toEqual([]);
const ownedLog = newLog();
await store.acquireOwnership({
logId: ownedLog,
harness: first,
ownerId: "a",
ttlMs: 1e4
});
await store.releaseOwnership({
logId: ownedLog,
ownerId: "a",
epoch: 1
});
await store.offerInput({
...offerOf(ownedLog, "i1"),
harness: other
});
expect(await orphans(other)).toEqual([]);
expect(await orphans(first)).toContainEqual({
logId: ownedLog,
harness: first,
reason: "offered"
});
});
it("notifies subscribers of a new offer", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const counter = countCalls(store, logId);
await store.offerInput(offerOf(logId, "i1"));
await expect.poll(counter.calls).toBe(1);
await store.offerInput(offerOf(logId, "i1"));
await store.offerInput(offerOf(logId, "i1", "other"));
await waitForLateCalls();
expect(counter.calls()).toBe(1);
counter.stop();
});
});
describe("log snapshots and lifecycle", () => {
const LIVE_MS = 1e4;
const recordsOf = (count) => Array.from({ length: count }, (_, index) => ({ type: `r${index + 1}` }));
const EPOCH = 1;
const logWith = async (store, count) => {
const logId = newLog();
await store.acquireOwnership({
logId,
harness: "h",
ownerId: "a",
ttlMs: LIVE_MS
});
await store.append({
logId,
seq: 1,
epoch: EPOCH,
records: recordsOf(count)
});
return logId;
};
const snapshotAt = (logId, seq) => ({
logId,
epoch: EPOCH,
seq,
version: "v1",
state: {
upTo: seq,
nested: { list: [seq] }
}
});
const logWithSnapshot = async (store) => {
const logId = await logWith(store, 4);
await store.putSnapshot(snapshotAt(logId, 3));
return logId;
};
const seqsAfter = async (store, logId, after) => (await store.read({
logId,
after
})).map((entry) => entry.seq);
it("stores the latest snapshot", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWith(store, 2);
expect(await store.getSnapshot({ logId })).toBeNull();
await store.putSnapshot(snapshotAt(logId, 1));
const latest = snapshotAt(logId, 2);
await store.putSnapshot(latest);
latest.state.nested.list.push(99);
required(await store.getSnapshot({ logId }), "the snapshot").seq = 7;
expect(await store.getSnapshot({ logId })).toEqual({
seq: 2,
version: "v1",
state: {
upTo: 2,
nested: { list: [2] }
}
});
});
it("fences putSnapshot and truncate", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const claim = {
logId,
harness: "h",
ttlMs: LIVE_MS
};
await store.acquireOwnership({
...claim,
ownerId: "a"
});
await store.releaseOwnership({
logId,
ownerId: "a",
epoch: 1
});
await store.acquireOwnership({
...claim,
ownerId: "b"
});
await store.append({
logId,
seq: 1,
epoch: 2,
records: recordsOf(2)
});
await store.putSnapshot({
...snapshotAt(logId, 1),
epoch: 2
});
await expect(store.putSnapshot({
...snapshotAt(logId, 2),
epoch: 1
})).rejects.toBeInstanceOf(OwnershipLostError);
await expect(store.truncate({
logId,
epoch: 1,
beforeSeq: 2
})).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.getSnapshot({ logId })).toMatchObject({ seq: 1 });
expect(await seqsAfter(store, logId, 0)).toEqual([1, 2]);
});
it("refuses putSnapshot and truncate on a log that no host acquired", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
await store.append({
logId,
seq: 1,
epoch: 0,
records: recordsOf(2)
});
await expect(store.putSnapshot({
...snapshotAt(logId, 1),
epoch: 0
})).rejects.toBeInstanceOf(OwnershipLostError);
await expect(store.truncate({
logId,
epoch: 0,
beforeSeq: 1
})).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.getSnapshot({ logId })).toBeNull();
});
it("refuses a snapshot past the head", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWith(store, 2);
await expect(store.putSnapshot(snapshotAt(logId, 5))).rejects.toBeInstanceOf(RangeError);
await expect(store.putSnapshot(snapshotAt(logId, 3))).rejects.toBeInstanceOf(RangeError);
expect(await store.getSnapshot({ logId })).toBeNull();
});
it("refuses a snapshot before the truncation point", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWithSnapshot(store);
await store.truncate({
logId,
epoch: EPOCH,
beforeSeq: 4
});
await expect(store.putSnapshot(snapshotAt(logId, 1))).rejects.toBeInstanceOf(RangeError);
expect(await store.getSnapshot({ logId })).toMatchObject({ seq: 3 });
await store.putSnapshot({
...snapshotAt(logId, 3),
version: "v2"
});
expect(await store.getSnapshot({ logId })).toMatchObject({
seq: 3,
version: "v2"
});
});
it("truncates before a snapshot and keeps counting positions", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWithSnapshot(store);
await store.truncate({
logId,
epoch: EPOCH,
beforeSeq: 4
});
expect(await store.read({
logId,
after: 3
})).toEqual([{
seq: 4,
record: { type: "r4" }
}]);
await store.append({
logId,
seq: 5,
epoch: EPOCH,
records: [{ type: "r5" }]
});
expect(await seqsAfter(store, logId, 3)).toEqual([4, 5]);
});
it("signals a read before the truncation point", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWithSnapshot(store);
await store.truncate({
logId,
epoch: EPOCH,
beforeSeq: 4
});
const read = store.read({
logId,
after: 0
});
await expect(read).rejects.toBeInstanceOf(LogTruncatedError);
await expect(read).rejects.toMatchObject({
logId,
firstSeq: 4
});
await expect(store.read({
logId,
after: 2
})).rejects.toBeInstanceOf(LogTruncatedError);
});
it("refuses to truncate past the snapshot", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWithSnapshot(store);
const bare = await logWith(store, 2);
await expect(store.truncate({
logId,
epoch: EPOCH,
beforeSeq: 5
})).rejects.toBeInstanceOf(RangeError);
await expect(store.truncate({
logId: bare,
epoch: EPOCH,
beforeSeq: 2
})).rejects.toBeInstanceOf(RangeError);
expect(await seqsAfter(store, logId, 0)).toEqual([
1,
2,
3,
4
]);
expect(await seqsAfter(store, bare, 0)).toEqual([1, 2]);
});
it("deletes a log and keeps its epoch counter", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const harness = newHarness();
const claim = {
logId,
harness,
ttlMs: LIVE_MS
};
const input = {
logId,
harness,
inputId: "i1",
record: { type: "in" }
};
await store.markHostAlive({
hostId: "host-1",
ttlMs: LIVE_MS
});
await store.acquireOwnership({
...claim,
ownerId: "a"
});
await store.pinToWorkers({
logId,
epoch: 1,
affinity: "host-1"
});
await store.append({
logId,
seq: 1,
epoch: 1,
records: recordsOf(2)
});
await store.putSnapshot({
...snapshotAt(logId, 1),
epoch: 1
});
await store.truncate({
logId,
epoch: 1,
beforeSeq: 2
});
await store.offerInput(input);
await store.deleteLog({ logId });
expect(await store.read({ logId })).toEqual([]);
expect(await store.listOfferedInputs({ logId })).toEqual([]);
expect(await store.getSnapshot({ logId })).toBeNull();
await expect(store.append({
logId,
seq: 1,
epoch: 1,
records: recordsOf(1)
})).rejects.toBeInstanceOf(OwnershipLostError);
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: recordsOf(1)
})).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.acquireOwnership({
...claim,
ownerId: "b"
})).toEqual({ epoch: 2 });
await store.append({
logId,
seq: 1,
epoch: 2,
records: recordsOf(1)
});
expect(await seqsAfter(store, logId, 0)).toEqual([1]);
expect(await store.offerInput(input)).toBe("offered");
const woken = newLog();
const orphans = () => store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
});
await store.acquireOwnership({
...claim,
logId: woken,
ownerId: "a"
});
await store.releaseOwnership({
logId: woken,
ownerId: "a",
epoch: 1,
wakeInMs: 0
});
expect(await orphans()).toEqual([{
logId: woken,
harness,
reason: "wake"
}]);
await store.deleteLog({ logId: woken });
expect(await orphans()).toEqual([]);
});
it("notifies subscribers of a delete", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWith(store, 1);
const counter = countCalls(store, logId);
await store.deleteLog({ logId });
await expect.poll(counter.calls).toBe(1);
counter.stop();
});
it("refuses a late offer to a deleted log until an acquire", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = await logWith(store, 1);
const harness = newHarness();
const input = {
logId,
harness,
inputId: "late",
record: { type: "in" }
};
await store.deleteLog({ logId });
expect(await store.offerInput(input)).toBe("deleted");
expect(await store.listOfferedInputs({ logId })).toEqual([]);
expect(await store.listLogsWithoutOwner({
limit: 10,
harnesses: [harness]
})).toEqual([]);
await store.acquireOwnership({
logId,
harness,
ownerId: "b",
ttlMs: LIVE_MS
});
expect(await store.offerInput(input)).toBe("offered");
});
it("refuses a late NO_OWNER_EPOCH write to a deleted log until an acquire", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const logId = newLog();
const writeAgain = (epoch) => store.append({
logId,
seq: 1,
epoch,
records: [{ type: "again" }]
});
await store.append({
logId,
seq: 1,
epoch: 0,
records: recordsOf(2)
});
await store.deleteLog({ logId });
await expect(writeAgain(0)).rejects.toBeInstanceOf(OwnershipLostError);
expect(await store.read({ logId })).toEqual([]);
expect(await store.acquireOwnership({
logId,
harness: "h",
ownerId: "a",
ttlMs: LIVE_MS
})).toEqual({ epoch: 1 });
await writeAgain(1);
expect(await store.read({ logId })).toEqual([{
seq: 1,
record: { type: "again" }
}]);
});
});
describe("log across instances", () => {
const twinLog = async () => {
const other = required((await resolveTwin()).stores.log, "the log store of the twin");
expect(other, "the twin must have a new log store instance").not.toBe(persistence.stores.log);
return other;
};
it("notifies a subscriber on the other instance", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const other = await twinLog();
const logId = newLog();
const counter = countCalls(other, logId);
await store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "from-a" }]
});
await expect.poll(counter.calls).toBe(1);
counter.stop();
expect(await other.read({ logId })).toEqual([{
seq: 1,
record: { type: "from-a" }
}]);
});
it("gives one owner across instances", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const other = await twinLog();
const claim = {
logId: newLog(),
harness: "h",
ttlMs: 1e4
};
expect(await store.acquireOwnership({
...claim,
ownerId: "a"
})).toEqual({ epoch: 1 });
expect(await other.acquireOwnership({
...claim,
ownerId: "b"
})).toBeNull();
});
it("deduplicates offers across instances", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const other = await twinLog();
const input = {
logId: newLog(),
harness: "h",
inputId: "i1",
record: {
type: "input",
text: "hi"
}
};
expect(await store.offerInput(input)).toBe("offered");
expect(await other.offerInput(input)).toBe("duplicate");
});
it("keeps notifying after a listener throws", async (ctx) => {
const store = persistence.stores.log;
if (!store) return ctx.skip("log store not provided");
const other = await twinLog();
const logId = newLog();
const errors = vi.spyOn(console, "error").mockImplementation(() => {});
ctx.onTestFinished(() => errors.mockRestore());
const stopThrower = store.subscribe({
logId,
listener: () => {
throw new Error("listener failed");
}
});
const counter = countCalls(other, logId);
await expect(store.append({
logId,
seq: 1,
epoch: 0,
records: [{ type: "one" }]
})).resolves.toBeUndefined();
await expect.poll(counter.calls).toBe(1);
await expect.poll(() => errors.mock.calls.length).toBeGreaterThan(0);
stopThrower();
counter.stop();
});
});
describe("sessions", () => {
const newId = () => `sessions-${crypto.randomUUID()}`;
const sessionEntry = (threadId, updatedAt, extra = {}) => ({
threadId,
members: [],
createdAt: 1,
updatedAt,
...extra
});
/** A new tenant, with an entry and a key factory for it. */
const newTenant = () => {
const tenantId = newId();
return {
tenantId,
entry: (threadId, updatedAt, extra = {}) => sessionEntry(threadId, updatedAt, {
tenantId,
...extra
}),
key: (threadId) => ({
tenantId,
threadId
})
};
};
const threadIds = (page) => page.entries.map((entry) => entry.threadId);
it("gets every field of an upserted entry, and undefined for an unknown thread", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, key } = newTenant();
const written = {
tenantId,
threadId: "thread",
members: [{
userId: "user-1",
role: "owner"
}, {
userId: "user-2",
role: "viewer"
}],
harness: "coder",
title: "Fix the build",
parentThreadId: "parent",
parentToolCallId: "call-1",
createdAt: 1,
updatedAt: 2,
pinned: true,
archived: true,
usage: {
turns: 2,
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
cachedTokens: 4,
cacheWriteTokens: 1,
cost: .25
},
metadata: { model: "m-1" }
};
await store.upsert(written);
expect(await store.get(key("thread"))).toEqual(written);
expect(await store.get(key("unknown"))).toBeUndefined();
});
it("keeps the owner and the members in getVersioned and list", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry, key } = newTenant();
const shared = entry("shared", 1, { members: [
{
userId: "alice",
role: "owner"
},
{
userId: "bob",
role: "member"
},
{
userId: "carol",
role: "viewer"
}
] });
await store.upsert(shared);
expect((await store.getVersioned(key("shared")))?.entry).toEqual(shared);
expect((await store.list({ tenantId })).entries).toEqual([shared]);
});
it("keeps one thread id in two tenants as two entries", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const threadId = newId();
const tenantId = newId();
const inTenant = sessionEntry(threadId, 10, {
tenantId,
title: "a"
});
const inDefault = sessionEntry(threadId, 20, { title: "b" });
await store.upsert(inTenant);
await store.upsert(inDefault);
expect(await store.get({
tenantId,
threadId
})).toEqual(inTenant);
expect(await store.get({ threadId })).toEqual(inDefault);
expect((await store.list({ tenantId })).entries).toEqual([inTenant]);
await store.delete({
tenantId,
threadId
});
expect(await store.get({
tenantId,
threadId
})).toBeUndefined();
expect(await store.get({ threadId })).toEqual(inDefault);
});
it("lists only the default tenant without tenantId", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const id = newId();
await store.upsert(sessionEntry(`${id}-tenant`, 10, { tenantId: id }));
await store.upsert(sessionEntry(`${id}-default`, 10));
const page = await store.list();
expect(threadIds(page)).toContain(`${id}-default`);
const otherTenants = page.entries.filter((entry) => (entry.tenantId ?? "") !== "");
expect(otherTenants).toEqual([]);
});
it("replaces the whole entry on a second upsert", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry, key } = newTenant();
await store.upsert(entry("thread", 2, {
title: "old",
pinned: true
}));
await store.upsert(entry("thread", 3, { title: "new" }));
expect(await store.get(key("thread"))).toEqual({
tenantId,
threadId: "thread",
members: [],
createdAt: 1,
updatedAt: 3,
title: "new"
});
});
it("lists newest first and pages with a cursor", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("c", 20));
await store.upsert(entry("a", 30));
await store.upsert(entry("b", 20));
expect(threadIds(await store.list({ tenantId }))).toEqual([
"a",
"b",
"c"
]);
const first = await store.list({
tenantId,
limit: 2
});
expect(threadIds(first)).toEqual(["a", "b"]);
expect(first.truncated).toBe(true);
const second = await store.list({
tenantId,
limit: 2,
cursor: required(first.cursor, "the cursor of a truncated page")
});
expect(threadIds(second)).toEqual(["c"]);
expect(second.truncated).toBeFalsy();
expect(second.cursor).toBeUndefined();
});
it("filters by parent thread", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("parent", 10));
await store.upsert(entry("child", 20, {
parentThreadId: "parent",
parentToolCallId: "call-1"
}));
await store.upsert(entry("sibling", 30, { parentThreadId: "parent" }));
await store.upsert(entry("other", 40, { parentThreadId: "other" }));
expect(threadIds(await store.list({
tenantId,
parentThreadId: "parent"
}))).toEqual(["sibling", "child"]);
});
it("lists only the entries with no parent for parentThreadId null", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("parent", 10));
await store.upsert(entry("child", 20, { parentThreadId: "parent" }));
expect(threadIds(await store.list({
tenantId,
parentThreadId: null
}))).toEqual(["parent"]);
});
it("filters by member id, with any role", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
const bob = {
userId: "bob",
role: "owner"
};
await store.upsert(entry("owned", 10, { members: [{
userId: "alice",
role: "owner"
}] }));
await store.upsert(entry("member", 20, { members: [bob, {
userId: "alice",
role: "member"
}] }));
await store.upsert(entry("viewer", 30, { members: [bob, {
userId: "alice",
role: "viewer"
}] }));
await store.upsert(entry("other", 40, { members: [bob] }));
await store.upsert(entry("child", 50, { parentThreadId: "owned" }));
expect(threadIds(await store.list({
tenantId,
memberId: "alice"
}))).toEqual([
"viewer",
"member",
"owned"
]);
});
it("filters by title search, without case", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("build", 10, { title: "Fix the Build" }));
await store.upsert(entry("docs", 20, { title: "Write docs" }));
await store.upsert(entry("untitled", 30));
expect(threadIds(await store.list({
tenantId,
search: "BUILD"
}))).toEqual(["build"]);
expect(threadIds(await store.list({
tenantId,
search: ""
}))).toEqual(["docs", "build"]);
});
it("filters by harness name", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("coder", 10, { harness: "coder" }));
await store.upsert(entry("writer", 20, { harness: "writer" }));
await store.upsert(entry("none", 30));
expect(threadIds(await store.list({
tenantId,
harness: "coder"
}))).toEqual(["coder"]);
});
it("filters by exact metadata values", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("both", 10, { metadata: {
project: "p-1",
branch: "main"
} }));
await store.upsert(entry("project-only", 20, { metadata: { project: "p-1" } }));
await store.upsert(entry("other", 30, { metadata: {
project: "p-2",
branch: "main"
} }));
await store.upsert(entry("none", 40));
expect(threadIds(await store.list({
tenantId,
metadata: { project: "p-1" }
}))).toEqual(["project-only", "both"]);
expect(threadIds(await store.list({
tenantId,
metadata: {
project: "p-1",
branch: "main"
}
}))).toEqual(["both"]);
});
it("combines search, harness, and metadata with AND", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
const base = {
title: "Fix the build",
harness: "coder",
metadata: { project: "p-1" }
};
await store.upsert(entry("match", 10, base));
await store.upsert(entry("wrong-title", 20, {
...base,
title: "Write docs"
}));
await store.upsert(entry("wrong-harness", 30, {
...base,
harness: "writer"
}));
await store.upsert(entry("wrong-project", 40, {
...base,
metadata: { project: "p-2" }
}));
expect(threadIds(await store.list({
tenantId,
search: "build",
harness: "coder",
metadata: { project: "p-1" }
}))).toEqual(["match"]);
});
it("hides archived entries unless withArchived is true", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry, key } = newTenant();
await store.upsert(entry("active", 10));
await store.upsert(entry("archived", 20, { archived: true }));
await store.upsert(entry("unarchived", 30, { archived: false }));
expect(threadIds(await store.list({ tenantId }))).toEqual(["unarchived", "active"]);
expect(threadIds(await store.list({
tenantId,
withArchived: true
}))).toEqual([
"unarchived",
"archived",
"active"
]);
expect(await store.get(key("archived"))).toEqual(entry("archived", 20, { archived: true }));
});
it("pages past archived entries without an empty page", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("a", 50));
await store.upsert(entry("hidden-1", 40, { archived: true }));
await store.upsert(entry("hidden-2", 30, { archived: true }));
await store.upsert(entry("b", 20));
const first = await store.list({
tenantId,
limit: 2
});
expect(threadIds(first)).toEqual(["a", "b"]);
expect(first.truncated).toBeFalsy();
});
it("pages the filtered list with a cursor", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry } = newTenant();
await store.upsert(entry("a", 50, { title: "build a" }));
await store.upsert(entry("skip", 40, { title: "docs" }));
await store.upsert(entry("b", 30, { title: "build b" }));
await store.upsert(entry("c", 20, { title: "build c" }));
const first = await store.list({
tenantId,
search: "build",
limit: 2
});
expect(threadIds(first)).toEqual(["a", "b"]);
expect(first.truncated).toBe(true);
const second = await store.list({
tenantId,
search: "build",
limit: 2,
cursor: required(first.cursor, "the cursor of a truncated page")
});
expect(threadIds(second)).toEqual(["c"]);
expect(second.truncated).toBeFalsy();
});
it("deletes an entry for good, and ignores an unknown thread", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { tenantId, entry, key } = newTenant();
await store.upsert(entry("kept", 1));
await store.upsert(entry("removed", 2, { title: "old" }));
await store.delete(key("removed"));
await store.delete(key("unknown"));
expect(await store.get(key("removed"))).toBeUndefined();
expect(await store.getVersioned(key("removed"))).toBeUndefined();
expect(threadIds(await store.list({ tenantId }))).toEqual(["kept"]);
const recreated = await store.upsertIf({
entry: entry("removed", 3),
revision: null
});
expect(recreated.ok).toBe(true);
expect(await store.get(key("removed"))).toEqual(entry("removed", 3));
});
it("writes with upsertIf only at the expected revision", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { entry, key } = newTenant();
expect(await store.getVersioned(key("thread"))).toBeUndefined();
const created = await store.upsertIf({
entry: entry("thread", 1),
revision: null
});
if (!created.ok) throw new Error("the first upsertIf conflicted");
expect(await store.upsertIf({
entry: entry("thread", 9),
revision: null
})).toEqual({
ok: false,
reason: "conflict"
});
expect(await store.getVersioned(key("thread"))).toEqual({
entry: entry("thread", 1),
revision: created.revision
});
const changed = await store.upsertIf({
entry: entry("thread", 2, { title: "new" }),
revision: created.revision
});
if (!changed.ok) throw new Error("upsertIf at the read revision conflicted");
expect(changed.revision).not.toBe(created.revision);
expect(await store.upsertIf({
entry: entry("thread", 3),
revision: created.revision
})).toEqual({
ok: false,
reason: "conflict"
});
expect(await store.get(key("thread"))).toEqual(entry("thread", 2, { title: "new" }));
});
it("changes the revision on a plain upsert, and never reuses one after delete", async (ctx) => {
const store = persistence.stores.sessions;
if (!store) return ctx.skip("sessions store not provided");
const { entry, key } = newTenant();
await store.upsert(entry("thread", 1));
const read = required(await store.getVersioned(key("thread")), "the versioned entry");
await store.upsert(entry("thread", 2));
expect(await store.upsertIf({
entry: entry("thread", 3),
revision: read.revision
})).toEqual({
ok: false,
reason: "conflict"
});
expect((await store.get(key("thread")))?.updatedAt).toBe(2);
const last = required(await store.getVersioned(key("thread")), "the versioned entry");
await store.delete(key("thread"));
expect(await store.upsertIf({
entry: entry("thread", 4),
revision: last.revision
})).toEqual({
ok: false,
reason: "conflict"
});
expect(await store.get(key("thread"))).toBeUndefined();
const recreated = await store.upsertIf({
entry: entry("thread", 5),
revision: null
});
if (!recreated.ok) throw new Error("upsertIf after delete conflicted");
expect(recreated.revision).not.toBe(last.revision);
});
});
describe("metadata", () => {
it("sets, gets, namespaces, and deletes without composite-key collisions", async (ctx) => {
const store = resolveStore("metadata");
if (!store) return ctx.skip("store not provided");
expect(await store.get("scope-a", "k")).toBeNull();
await store.set("scope-a", "k", { n: 1 });
await store.set("scope-b", "k", { n: 2 });
expect(await store.get("scope-a", "k")).toEqual({ n: 1 });
expect(await store.get("scope-b", "k")).toEqual({ n: 2 });
await store.set("scope-a", "k", { n: 3 });
expect(await store.get("scope-a", "k")).toEqual({ n: 3 });
await store.delete("scope-a", "k");
expect(await store.get("scope-a", "k")).toBeNull();
expect(await store.get("scope-b", "k")).toEqual({ n: 2 });
await store.set("a:b", "c", "left");
await store.set("a", "b:c", "right");
expect(await store.get("a:b", "c")).toBe("left");
expect(await store.get("a", "b:c")).toBe("right");
await store.delete("a:b", "c");
expect(await store.get("a:b", "c")).toBeNull();
expect(await store.get("a", "b:c")).toBe("right");
});
it("setIf writes only when the revision matches", async (ctx) => {
const store = resolveStore("metadata");
if (!store?.setIf || !store.getVersioned) return ctx.skip("metadata setIf not provided");
expect(await store.getVersioned("cas", "k")).toBeNull();
expect(await store.setIf("cas", "absent", { n: 0 }, "no-such")).toEqual({
ok: false,
reason: "conflict"
});
expect(await store.get("cas", "absent")).toBeNull();
const created = await store.setIf("cas", "k", { n: 1 }, null);
expect(created.ok).toBe(true);
if (!created.ok) return;
expect((await store.getVersioned("cas", "k"))?.revision).toBe(created.revision);
expect(await store.setIf("cas", "k", { n: 9 }, null)).toEqual({
ok: false,
reason: "conflict"
});
const current = await store.getVersioned("cas", "k");
expect(current?.value).toEqual({ n: 1 });
const revision = current?.revision ?? null;
const updated = await store.setIf("cas", "k", { n: 2 }, revision);
expect(updated.ok).toBe(true);
if (!updated.ok) return;
expect((await store.getVersioned("cas", "k"))?.revision).toBe(updated.revision);
expect(await store.setIf("cas", "k", { n: 3 }, revision)).toEqual({
ok: false,
reason: "conflict"
});
expect(await store.get("cas", "k")).toEqual({ n: 2 });
const before = await store.getVersioned("cas", "k");
await store.set("cas", "k", { n: 4 });
const stale = before?.revision ?? null;
expect((await store.setIf("cas", "k", { n: 5 }, stale)).ok).toBe(false);
expect(await store.get("cas", "k")).toEqual({ n: 4 });
const old = (await store.getVersioned("cas", "k"))?.revision ?? null;
await store.delete("cas", "k");
expect(await store.getVersioned("cas", "k")).toBeNull();
expect(await store.setIf("cas", "k", { n: 7 }, old)).toEqual({
ok: false,
reason: "conflict"
});
expect(await store.get("cas", "k")).toBeNull();
const recreated = await store.setIf("cas", "k", { n: 6 }, null);
expect(recreated.ok).toBe(true);
if (!recreated.ok) return;
expect(recreated.revision).not.toBe(old);
expect(await store.get("cas", "k")).toEqual({ n: 6 });
});
});
});
}
//#endregion
export { runPersistenceConformance };
//# sourceMappingURL=conformance.js.map