@fedify/fedify
Version:
An ActivityPub server framework
447 lines (446 loc) • 14.6 kB
JavaScript
import { Temporal } from "@js-temporal/polyfill";
import "urlpattern-polyfill";
globalThis.addEventListener = () => {};
import { t as assertEquals } from "../assert_equals-C-ZRDbaf.mjs";
import "../std__assert-BBjXFNOb.mjs";
import { t as assertThrows } from "../assert_throws-BOkhLGYc.mjs";
import { t as MemoryKvStore } from "../kv-rV3vodCc.mjs";
import { n as normalizeCircuitBreakerOptions, r as parseCircuitBreakerKvState, t as CircuitBreaker } from "../circuit-breaker-CSWsyoef.mjs";
import { test } from "@fedify/fixture";
//#region src/federation/circuit-breaker.test.ts
var AlwaysConflictingKvStore = class extends MemoryKvStore {
attempts = 0;
cas(_key, _expectedValue, _newValue, _options) {
this.attempts++;
if (this.attempts > 10) throw new Error("beforeSend did not stop retrying CAS misses");
return Promise.resolve(false);
}
};
var CountingCasKvStore = class extends MemoryKvStore {
attempts = 0;
cas(key, expectedValue, newValue, options) {
this.attempts++;
return super.cas(key, expectedValue, newValue, options);
}
};
test("normalizeCircuitBreakerOptions() uses numeric failure policy", () => {
const options = normalizeCircuitBreakerOptions({
failureThreshold: 3,
failureWindow: { minutes: 10 }
});
const failures = [
Temporal.Instant.from("2026-05-25T00:00:00Z"),
Temporal.Instant.from("2026-05-25T00:05:00Z"),
Temporal.Instant.from("2026-05-25T00:10:00Z")
];
assertEquals(options.failure(failures.slice(0, 2)), false);
assertEquals(options.failure(failures), true);
assertEquals(options.failure([
Temporal.Instant.from("2026-05-25T00:00:00Z"),
Temporal.Instant.from("2026-05-25T00:11:00Z"),
Temporal.Instant.from("2026-05-25T00:12:00Z")
]), false);
assertEquals(options.pruneFailures([
Temporal.Instant.from("2026-05-25T00:00:00Z"),
Temporal.Instant.from("2026-05-25T00:09:00Z"),
Temporal.Instant.from("2026-05-25T00:10:00Z"),
Temporal.Instant.from("2026-05-25T00:11:00Z"),
Temporal.Instant.from("2026-05-25T00:12:00Z")
], Temporal.Instant.from("2026-05-25T00:12:00Z")).map((t) => t.toString()), [
"2026-05-25T00:10:00Z",
"2026-05-25T00:11:00Z",
"2026-05-25T00:12:00Z"
]);
});
test("normalizeCircuitBreakerOptions() validates numeric failure policy", () => {
assertThrows(() => normalizeCircuitBreakerOptions({ failureThreshold: 0 }), TypeError, "failureThreshold");
assertThrows(() => normalizeCircuitBreakerOptions({ failureThreshold: 1.5 }), TypeError, "failureThreshold");
});
test("normalizeCircuitBreakerOptions() truncates sub-millisecond durations", () => {
assertEquals(normalizeCircuitBreakerOptions({ recoveryDelay: {
milliseconds: 1,
nanoseconds: 5e5
} }).recoveryDelay, Temporal.Duration.from({ milliseconds: 1 }));
});
test("normalizeCircuitBreakerOptions() validates positive durations", () => {
assertThrows(() => normalizeCircuitBreakerOptions({ recoveryDelay: { seconds: 0 } }), RangeError, "recoveryDelay");
assertThrows(() => normalizeCircuitBreakerOptions({ heldActivityTtl: { seconds: 0 } }), RangeError, "heldActivityTtl");
assertThrows(() => normalizeCircuitBreakerOptions({ failureWindow: { seconds: 0 } }), RangeError, "failureWindow");
assertThrows(() => normalizeCircuitBreakerOptions({ releaseInterval: { seconds: 0 } }), RangeError, "releaseInterval");
assertThrows(() => normalizeCircuitBreakerOptions({ releaseInterval: { nanoseconds: 5e5 } }), RangeError, "releaseInterval");
});
test("normalizeCircuitBreakerOptions() accepts callback failure policy", () => {
const options = normalizeCircuitBreakerOptions({ failure: (timestamps) => timestamps.length >= 2 });
const base = Temporal.Instant.from("2026-05-25T00:00:00Z");
const failures = Array.from({ length: 105 }, (_, i) => base.add({ minutes: i }));
assertEquals(options.failure([Temporal.Instant.from("2026-05-25T00:00:00Z")]), false);
assertEquals(options.failure([Temporal.Instant.from("2026-05-25T00:00:00Z"), Temporal.Instant.from("2026-05-25T00:01:00Z")]), true);
assertEquals(options.pruneFailures(failures, base.add({ minutes: 105 })).map((t) => t.toString()), failures.slice(-100).map((t) => t.toString()));
});
test("parseCircuitBreakerKvState() validates stored shape", () => {
assertEquals(parseCircuitBreakerKvState({
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z",
halfOpened: "2026-05-25T00:00:00Z"
}), {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z",
halfOpened: "2026-05-25T00:00:00Z"
});
assertEquals(parseCircuitBreakerKvState({ state: "open" }), void 0);
assertEquals(parseCircuitBreakerKvState({
state: "other",
failures: []
}), void 0);
assertEquals(parseCircuitBreakerKvState({
state: "open",
failures: [],
opened: 1
}), void 0);
assertEquals(parseCircuitBreakerKvState({
state: "open",
failures: ["not an instant"]
}), void 0);
assertEquals(parseCircuitBreakerKvState({
state: "open",
failures: [],
halfOpened: "not an instant"
}), void 0);
});
test("CircuitBreaker opens, probes, closes, and drops held activities", async () => {
const kv = new MemoryKvStore();
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
const transitions = [];
const circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: {
failureThreshold: 2,
failureWindow: { minutes: 10 },
recoveryDelay: { minutes: 30 },
heldActivityTtl: { days: 7 },
onStateChange(host, previousState, newState) {
transitions.push(`${host}:${previousState}->${newState}`);
}
}
});
await circuit.recordFailure("remote.example");
assertEquals(await circuit.getState("remote.example"), {
state: "closed",
failures: ["2026-05-25T00:00:00Z"]
});
now = Temporal.Instant.from("2026-05-25T00:05:00Z");
await circuit.recordFailure("remote.example");
assertEquals(await circuit.getState("remote.example"), {
state: "open",
failures: ["2026-05-25T00:00:00Z", "2026-05-25T00:05:00Z"],
opened: "2026-05-25T00:05:00Z"
});
assertEquals(transitions, ["remote.example:closed->open"]);
let decision = await circuit.beforeSend("remote.example", {});
assertEquals(decision, {
type: "hold",
delay: Temporal.Duration.from({ minutes: 30 }),
heldSince: now,
state: "open"
});
now = Temporal.Instant.from("2026-05-25T00:35:00Z");
decision = await circuit.beforeSend("remote.example", {});
assertEquals(decision, {
type: "send",
probe: true,
stateChange: {
previousState: "open",
newState: "half-open"
}
});
assertEquals(await circuit.getState("remote.example"), {
state: "half-open",
failures: ["2026-05-25T00:00:00Z", "2026-05-25T00:05:00Z"],
opened: "2026-05-25T00:05:00Z",
halfOpened: "2026-05-25T00:35:00Z"
});
await circuit.recordSuccess("remote.example");
assertEquals(await circuit.getState("remote.example"), void 0);
assertEquals(transitions, [
"remote.example:closed->open",
"remote.example:open->half-open",
"remote.example:half-open->closed"
]);
decision = await circuit.beforeSend("remote.example", { circuitHeldSince: "2026-05-17T00:00:00Z" });
assertEquals(decision, {
type: "drop",
heldSince: Temporal.Instant.from("2026-05-17T00:00:00Z")
});
await kv.set([
"_fedify",
"circuit",
"remote.example"
], {
state: "open",
failures: ["2026-05-25T00:00:00Z", "2026-05-25T00:05:00Z"],
opened: "2026-05-25T00:05:00Z"
});
decision = await circuit.beforeSend("remote.example", { circuitHeldSince: "2026-05-17T00:00:00Z" });
assertEquals(decision, {
type: "drop",
heldSince: Temporal.Instant.from("2026-05-17T00:00:00Z")
});
});
test("CircuitBreaker recovers stale half-open probes", async () => {
const kv = new MemoryKvStore();
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
const circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: {
recoveryDelay: { seconds: 30 },
releaseInterval: { seconds: 5 }
}
});
await kv.set([
"_fedify",
"circuit",
"remote.example"
], {
state: "half-open",
failures: ["2026-05-24T23:00:00Z"],
opened: "2026-05-24T23:00:00Z",
halfOpened: "2026-05-24T23:59:54Z"
});
let decision = await circuit.beforeSend("remote.example", {});
assertEquals(decision, {
type: "hold",
state: "half-open",
delay: Temporal.Duration.from({ seconds: 5 }),
heldSince: now
});
assertEquals(await circuit.getState("remote.example"), {
state: "half-open",
failures: ["2026-05-24T23:00:00Z"],
opened: "2026-05-24T23:00:00Z",
halfOpened: "2026-05-24T23:59:54Z"
});
now = Temporal.Instant.from("2026-05-25T00:00:30Z");
decision = await circuit.beforeSend("remote.example", {});
assertEquals(decision, {
type: "send",
probe: true
});
assertEquals(await circuit.getState("remote.example"), {
state: "half-open",
failures: ["2026-05-24T23:00:00Z"],
opened: "2026-05-24T23:00:00Z",
halfOpened: "2026-05-25T00:00:30Z"
});
});
test("CircuitBreaker caps held delays at activity TTL", async () => {
const kv = new MemoryKvStore();
const now = Temporal.Instant.from("2026-05-25T00:05:00Z");
const circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: {
recoveryDelay: { minutes: 30 },
heldActivityTtl: { minutes: 10 },
releaseInterval: { minutes: 10 }
}
});
await kv.set([
"_fedify",
"circuit",
"new-open.example"
], {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z"
});
let decision = await circuit.beforeSend("new-open.example", {});
assertEquals(decision.type, "hold");
if (decision.type === "hold") {
assertEquals(decision.state, "open");
assertEquals(decision.delay.total({ unit: "minute" }), 10);
assertEquals(decision.heldSince.toString(), "2026-05-25T00:05:00Z");
}
await kv.set([
"_fedify",
"circuit",
"open.example"
], {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z"
});
decision = await circuit.beforeSend("open.example", { circuitHeldSince: "2026-05-25T00:00:00Z" });
assertEquals(decision.type, "hold");
if (decision.type === "hold") {
assertEquals(decision.state, "open");
assertEquals(decision.delay.total({ unit: "minute" }), 5);
assertEquals(decision.heldSince.toString(), "2026-05-25T00:00:00Z");
}
await kv.set([
"_fedify",
"circuit",
"half-open.example"
], {
state: "half-open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z",
halfOpened: "2026-05-25T00:00:00Z"
});
decision = await circuit.beforeSend("half-open.example", { circuitHeldSince: "2026-05-25T00:00:00Z" });
assertEquals(decision.type, "hold");
if (decision.type === "hold") {
assertEquals(decision.state, "half-open");
assertEquals(decision.delay.total({ unit: "minute" }), 5);
assertEquals(decision.heldSince.toString(), "2026-05-25T00:00:00Z");
}
});
test("CircuitBreaker ignores malformed held timestamps", async () => {
const kv = new MemoryKvStore();
const now = Temporal.Instant.from("2026-05-25T00:05:00Z");
const circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: { recoveryDelay: { minutes: 30 } }
});
await kv.set([
"_fedify",
"circuit",
"malformed-held.example"
], {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z"
});
assertEquals(await circuit.beforeSend("malformed-held.example", { circuitHeldSince: "not an instant" }), {
type: "hold",
state: "open",
delay: Temporal.Duration.from({ minutes: 25 }),
heldSince: now
});
});
test("CircuitBreaker bounds beforeSend CAS retries", async () => {
let kv = new AlwaysConflictingKvStore();
const now = Temporal.Instant.from("2026-05-25T00:30:00Z");
let circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: {
recoveryDelay: { minutes: 30 },
releaseInterval: { seconds: 5 }
}
});
await kv.set([
"_fedify",
"circuit",
"open.example"
], {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z"
});
let decision = await circuit.beforeSend("open.example", {});
assertEquals(kv.attempts, 10);
assertEquals(decision, {
type: "hold",
state: "open",
delay: Temporal.Duration.from({ seconds: 5 }),
heldSince: now
});
kv = new AlwaysConflictingKvStore();
circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: {
recoveryDelay: { minutes: 30 },
releaseInterval: { seconds: 5 }
}
});
await kv.set([
"_fedify",
"circuit",
"half-open.example"
], {
state: "half-open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z",
halfOpened: "2026-05-25T00:00:00Z"
});
decision = await circuit.beforeSend("half-open.example", {});
assertEquals(kv.attempts, 10);
assertEquals(decision, {
type: "hold",
state: "half-open",
delay: Temporal.Duration.from({ seconds: 5 }),
heldSince: now
});
});
test("CircuitBreaker skips recording failures for open circuits", async () => {
const kv = new CountingCasKvStore();
const circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => Temporal.Instant.from("2026-05-25T00:01:00Z")
});
await kv.set([
"_fedify",
"circuit",
"open.example"
], {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z"
});
assertEquals(await circuit.recordFailure("open.example"), void 0);
assertEquals(kv.attempts, 0);
assertEquals(await kv.get([
"_fedify",
"circuit",
"open.example"
]), {
state: "open",
failures: ["2026-05-25T00:00:00Z"],
opened: "2026-05-25T00:00:00Z"
});
});
test("CircuitBreaker prunes stale closed failure history", async () => {
const kv = new MemoryKvStore();
let now = Temporal.Instant.from("2026-05-25T00:00:00Z");
const circuit = new CircuitBreaker({
kv,
prefix: ["_fedify", "circuit"],
now: () => now,
options: {
failureThreshold: 2,
failureWindow: { minutes: 10 }
}
});
await circuit.recordFailure("sporadic.example");
assertEquals(await circuit.getState("sporadic.example"), {
state: "closed",
failures: ["2026-05-25T00:00:00Z"]
});
now = Temporal.Instant.from("2026-05-25T00:20:00Z");
await circuit.recordFailure("sporadic.example");
assertEquals(await circuit.getState("sporadic.example"), {
state: "closed",
failures: ["2026-05-25T00:20:00Z"]
});
now = Temporal.Instant.from("2026-05-25T00:40:00Z");
await circuit.recordFailure("sporadic.example");
assertEquals(await circuit.getState("sporadic.example"), {
state: "closed",
failures: ["2026-05-25T00:40:00Z"]
});
});
//#endregion
export {};