@fedify/fedify
Version:
An ActivityPub server framework
338 lines (337 loc) • 11.2 kB
JavaScript
import { Temporal } from "@js-temporal/polyfill";
import "urlpattern-polyfill";
globalThis.addEventListener = () => {};
import { getLogger } from "@logtape/logtape";
//#region src/federation/circuit-breaker.ts
const MAX_CUSTOM_FAILURE_HISTORY = 100;
/**
* Tracks reachability state for remote outbox delivery hosts.
* @since 2.3.0
*/
var CircuitBreaker = class {
#kv;
#prefix;
#options;
#now;
#stateChangeObserver;
constructor(options) {
this.#kv = options.kv;
this.#prefix = options.prefix;
this.#options = normalizeCircuitBreakerOptions(options.options ?? {});
this.#now = options.now ?? (() => Temporal.Now.instant());
this.#stateChangeObserver = options.stateChangeObserver;
}
get options() {
return this.#options;
}
capHeldDelay(heldSince, delay) {
const now = this.#now();
return now.until(this.#capHeldRetryAt(now, heldSince, now.add(delay)));
}
async beforeSend(remoteHost, message) {
const heldSince = parseHeldSince(message.circuitHeldSince);
const now = this.#now();
if (heldSince != null && Temporal.Instant.compare(heldSince.add(this.#options.heldActivityTtl), now) <= 0) return {
type: "drop",
heldSince
};
let lastConflictingState;
for (let attempt = 0; attempt < 10; attempt++) {
const oldState = await this.#get(remoteHost);
if (oldState == null || oldState.state === "closed") return {
type: "send",
probe: false
};
if (oldState.state === "half-open") {
const halfOpened = oldState.halfOpened == null ? void 0 : Temporal.Instant.from(oldState.halfOpened);
if (halfOpened != null) {
const staleAt = halfOpened.add(this.#options.recoveryDelay);
if (Temporal.Instant.compare(now, staleAt) < 0) {
const releaseAt = now.add(this.#options.releaseInterval);
const retryAt = Temporal.Instant.compare(releaseAt, staleAt) < 0 ? releaseAt : staleAt;
const cappedRetryAt = this.#capHeldRetryAt(now, heldSince, retryAt);
return {
type: "hold",
state: "half-open",
delay: now.until(cappedRetryAt),
heldSince: heldSince ?? now
};
}
}
const newState = {
...oldState,
state: "half-open",
halfOpened: now.toString()
};
if (await this.#replace(remoteHost, oldState, newState)) return {
type: "send",
probe: true
};
lastConflictingState = "half-open";
continue;
}
const probeAt = (oldState.opened == null ? now : Temporal.Instant.from(oldState.opened)).add(this.#options.recoveryDelay);
if (Temporal.Instant.compare(now, probeAt) < 0) {
const retryAt = this.#capHeldRetryAt(now, heldSince, probeAt);
return {
type: "hold",
state: "open",
delay: now.until(retryAt),
heldSince: heldSince ?? now
};
}
const newState = {
...oldState,
state: "half-open",
halfOpened: now.toString()
};
if (await this.#replace(remoteHost, oldState, newState)) {
await this.#notifyStateChange(remoteHost, "open", "half-open");
return {
type: "send",
probe: true,
stateChange: {
previousState: "open",
newState: "half-open"
}
};
}
lastConflictingState = "open";
}
if (lastConflictingState != null) {
const retryAt = this.#capHeldRetryAt(now, heldSince, now.add(this.#options.releaseInterval));
return {
type: "hold",
state: lastConflictingState,
delay: now.until(retryAt),
heldSince: heldSince ?? now
};
}
throw new Error(`Failed to update circuit breaker state for ${remoteHost}`);
}
async recordSuccess(remoteHost) {
for (let attempt = 0; attempt < 10; attempt++) {
const oldState = await this.#get(remoteHost);
if (oldState == null) return void 0;
if (await this.#replace(remoteHost, oldState, void 0)) {
if (oldState.state !== "closed") {
await this.#notifyStateChange(remoteHost, oldState.state, "closed");
return {
previousState: oldState.state,
newState: "closed"
};
}
return;
}
}
throw new Error(`Failed to update circuit breaker state for ${remoteHost}`);
}
async recordReachableFailure(remoteHost) {
return await this.recordSuccess(remoteHost);
}
async recordFailure(remoteHost) {
const now = this.#now();
for (let attempt = 0; attempt < 10; attempt++) {
const oldState = await this.#get(remoteHost);
if (oldState?.state === "open") return void 0;
const oldFailures = oldState?.failures.map(Temporal.Instant.from) ?? [];
const failures = this.#options.pruneFailures([...oldFailures, now], now);
let newState;
let transition;
if (oldState?.state === "half-open" || this.#options.failure(failures)) {
newState = {
state: "open",
failures: failures.map((t) => t.toString()),
opened: now.toString()
};
transition = [oldState?.state ?? "closed", "open"];
} else newState = {
state: "closed",
failures: failures.map((t) => t.toString())
};
if (await this.#replace(remoteHost, oldState, newState)) {
if (transition != null) {
await this.#notifyStateChange(remoteHost, transition[0], transition[1]);
return {
previousState: transition[0],
newState: transition[1]
};
}
return;
}
}
throw new Error(`Failed to update circuit breaker state for ${remoteHost}`);
}
async dropActivity(remoteHost, details) {
try {
await this.#options.onActivityDrop?.(remoteHost, details);
} catch (error) {
getLogger([
"fedify",
"federation",
"circuit"
]).error("An unexpected error occurred in circuit breaker activity drop handler:\n{error}", {
remoteHost,
error
});
}
}
async getState(remoteHost) {
return await this.#get(remoteHost);
}
#key(remoteHost) {
return [...this.#prefix, remoteHost];
}
#capHeldRetryAt(now, heldSince, retryAt) {
const expiresAt = (heldSince ?? now).add(this.#options.heldActivityTtl);
return Temporal.Instant.compare(expiresAt, retryAt) < 0 ? expiresAt : retryAt;
}
async #get(remoteHost) {
return parseCircuitBreakerKvState(await this.#kv.get(this.#key(remoteHost)));
}
async #replace(remoteHost, oldState, newState) {
const key = this.#key(remoteHost);
if (this.#kv.cas == null) {
if (newState == null) await this.#kv.delete(key);
else await this.#kv.set(key, newState);
return true;
}
return await this.#kv.cas(key, oldState, newState);
}
async #notifyStateChange(remoteHost, previousState, newState) {
try {
await this.#options.onStateChange?.(remoteHost, previousState, newState);
} catch (error) {
getLogger([
"fedify",
"federation",
"circuit"
]).error("An unexpected error occurred in circuit breaker state change handler:\n{error}", {
remoteHost,
previousState,
newState,
error
});
}
try {
await this.#stateChangeObserver?.(remoteHost, previousState, newState);
} catch (error) {
getLogger([
"fedify",
"federation",
"circuit"
]).error("An unexpected error occurred in circuit breaker state change observer:\n{error}", {
remoteHost,
previousState,
newState,
error
});
}
}
};
/**
* Normalizes user-provided circuit breaker options into the internal policy
* shape used while processing queued outbox deliveries.
*
* @param options The public circuit breaker options supplied to Fedify.
* @returns The normalized failure predicate, failure pruning function,
* duration values, and optional callbacks with defaults applied.
* @throws {RangeError} If any configured duration is not positive.
* @throws {TypeError} If `failureThreshold` is not a positive integer.
*/
function normalizeCircuitBreakerOptions(options) {
const recoveryDelay = toInstantDuration(options.recoveryDelay ?? { minutes: 30 });
const heldActivityTtl = toInstantDuration(options.heldActivityTtl ?? { hours: 168 });
const releaseInterval = toInstantDuration(options.releaseInterval ?? { seconds: 1 });
assertPositiveDuration(recoveryDelay, "recoveryDelay");
assertPositiveDuration(heldActivityTtl, "heldActivityTtl");
assertPositiveDuration(releaseInterval, "releaseInterval");
let failure;
let pruneFailures;
if (options.failure == null) {
const failureThreshold = options.failureThreshold ?? 5;
if (!Number.isInteger(failureThreshold) || failureThreshold <= 0) throw new TypeError("failureThreshold must be a positive integer.");
const failureWindow = toInstantDuration(options.failureWindow ?? { minutes: 10 });
assertPositiveDuration(failureWindow, "failureWindow");
pruneFailures = (timestamps, now) => {
const earliest = now.subtract(failureWindow);
return timestamps.filter((timestamp) => Temporal.Instant.compare(timestamp, earliest) >= 0).slice(-failureThreshold);
};
failure = (timestamps) => {
if (timestamps.length < failureThreshold) return false;
const first = timestamps[timestamps.length - failureThreshold];
const last = timestamps[timestamps.length - 1];
return Temporal.Duration.compare(first.until(last), failureWindow) <= 0;
};
} else {
failure = options.failure;
pruneFailures = (timestamps) => timestamps.slice(-MAX_CUSTOM_FAILURE_HISTORY);
}
return {
failure,
pruneFailures,
recoveryDelay,
heldActivityTtl,
releaseInterval,
onStateChange: options.onStateChange,
onActivityDrop: options.onActivityDrop
};
}
function toInstantDuration(duration) {
const parsed = Temporal.Duration.from(duration);
return Temporal.Duration.from({ milliseconds: Math.trunc(parsed.total({
unit: "millisecond",
relativeTo: Temporal.PlainDateTime.from("2026-01-01T00:00:00")
})) });
}
function assertPositiveDuration(duration, name) {
if (Temporal.Duration.compare(duration, { seconds: 0 }) <= 0) throw new RangeError(`${name} must be a positive duration.`);
}
function parseHeldSince(value) {
if (value == null) return void 0;
try {
return Temporal.Instant.from(value);
} catch (error) {
getLogger([
"fedify",
"federation",
"circuit"
]).warn("Invalid circuitHeldSince value in queued outbox message: {value}", {
value,
error
});
return;
}
}
/**
* Parses a value loaded from the circuit breaker KV store.
*
* @param value The raw KV value to validate.
* @returns A circuit breaker state when `value` has a recognized state and
* valid instant strings, or `undefined` when the stored value is malformed.
*/
function parseCircuitBreakerKvState(value) {
const isInstantString = (v) => {
if (typeof v !== "string") return false;
try {
Temporal.Instant.from(v);
return true;
} catch {
return false;
}
};
if (typeof value !== "object" || value == null) return void 0;
const record = value;
if (record.state !== "closed" && record.state !== "open" && record.state !== "half-open") return;
if (!Array.isArray(record.failures) || !record.failures.every((failure) => isInstantString(failure))) return;
if (record.opened != null && !isInstantString(record.opened)) return;
if (record.halfOpened != null && !isInstantString(record.halfOpened)) return;
return {
state: record.state,
failures: record.failures,
...record.opened == null ? {} : { opened: record.opened },
...record.halfOpened == null ? {} : { halfOpened: record.halfOpened }
};
}
//#endregion
export { normalizeCircuitBreakerOptions as n, parseCircuitBreakerKvState as r, CircuitBreaker as t };