@fedify/fedify
Version:
An ActivityPub server framework
331 lines (326 loc) • 11.5 kB
JavaScript
import { Temporal } from "@js-temporal/polyfill";
import { URLPattern } from "urlpattern-polyfill";
globalThis.addEventListener = () => {};
import { assertEquals } from "../assert_equals-CTYbeopb.js";
import { assert } from "../assert-DmFG7ppO.js";
import "../assert_instance_of-CF09JHYM.js";
import { test } from "../testing-BZ0dJ4qn.js";
import { assertGreater, assertGreaterOrEqual } from "../std__assert-vp0TKMS1.js";
import { assertFalse } from "../assert_rejects-C-sxEMM5.js";
import "../assert_is_error-nrwA1GeT.js";
import "../assert_not_equals-Dc7y-V5Q.js";
import "../assert_throws-Cn9C6Jur.js";
import { delay } from "@es-toolkit/es-toolkit";
//#region federation/mq.ts
/**
* A message queue that processes messages in the same process.
* Do not use this in production as it does neither persist messages nor
* distribute them across multiple processes.
*
* @since 0.5.0
*/
var InProcessMessageQueue = class {
#messages;
#monitors;
#pollIntervalMs;
/**
* In-process message queue does not provide native retry mechanisms.
* @since 1.7.0
*/
nativeRetrial = false;
/**
* Constructs a new {@link InProcessMessageQueue} with the given options.
* @param options Additional options for the in-process message queue.
*/
constructor(options = {}) {
this.#messages = [];
this.#monitors = {};
this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 5 }).total("millisecond");
}
enqueue(message, options) {
const delay$1 = options?.delay == null ? 0 : Math.max(options.delay.total("millisecond"), 0);
if (delay$1 > 0) {
setTimeout(() => this.enqueue(message, {
...options,
delay: void 0
}), delay$1);
return Promise.resolve();
}
this.#messages.push(message);
for (const monitorId in this.#monitors) this.#monitors[monitorId]();
return Promise.resolve();
}
enqueueMany(messages, options) {
if (messages.length === 0) return Promise.resolve();
const delay$1 = options?.delay == null ? 0 : Math.max(options.delay.total("millisecond"), 0);
if (delay$1 > 0) {
setTimeout(() => this.enqueueMany(messages, {
...options,
delay: void 0
}), delay$1);
return Promise.resolve();
}
this.#messages.push(...messages);
for (const monitorId in this.#monitors) this.#monitors[monitorId]();
return Promise.resolve();
}
async listen(handler, options = {}) {
const signal = options.signal;
while (signal == null || !signal.aborted) {
while (this.#messages.length > 0) {
const message = this.#messages.shift();
await handler(message);
}
await this.#wait(this.#pollIntervalMs, signal);
}
}
#wait(ms, signal) {
let timer = null;
return Promise.any([new Promise((resolve) => {
signal?.addEventListener("abort", () => {
if (timer != null) clearTimeout(timer);
resolve();
}, { once: true });
const monitorId = crypto.randomUUID();
this.#monitors[monitorId] = () => {
delete this.#monitors[monitorId];
if (timer != null) clearTimeout(timer);
resolve();
};
}), new Promise((resolve) => timer = setTimeout(resolve, ms))]);
}
};
/**
* A message queue that processes messages in parallel. It takes another
* {@link MessageQueue}, and processes messages in parallel up to a certain
* number of workers.
*
* Actually, it's rather a decorator than a queue itself.
*
* Note that the workers do not run in truly parallel, in the sense that they
* are not running in separate threads or processes. They are running in the
* same process, but are scheduled to run in parallel. Hence, this is useful
* for I/O-bound tasks, but not for CPU-bound tasks, which is okay for Fedify's
* workloads.
*
* @since 1.0.0
*/
var ParallelMessageQueue = class ParallelMessageQueue {
queue;
workers;
/**
* Inherits the native retry capability from the wrapped queue.
* @since 1.7.0
*/
nativeRetrial;
/**
* Constructs a new {@link ParallelMessageQueue} with the given queue and
* number of workers.
* @param queue The message queue to use under the hood. Note that
* {@link ParallelMessageQueue} cannot be nested.
* @param workers The number of workers to process messages in parallel.
* @throws {TypeError} If the given queue is an instance of
* {@link ParallelMessageQueue}.
*/
constructor(queue, workers) {
if (queue instanceof ParallelMessageQueue) throw new TypeError("Cannot nest ParallelMessageQueue.");
this.queue = queue;
this.workers = workers;
this.nativeRetrial = queue.nativeRetrial;
}
enqueue(message, options) {
return this.queue.enqueue(message, options);
}
async enqueueMany(messages, options) {
if (this.queue.enqueueMany == null) {
const results = await Promise.allSettled(messages.map((message) => this.queue.enqueue(message, options)));
const errors = results.filter((r) => r.status === "rejected").map((r) => r.reason);
if (errors.length > 1) throw new AggregateError(errors, "Failed to enqueue messages.");
else if (errors.length === 1) throw errors[0];
return;
}
await this.queue.enqueueMany(messages, options);
}
listen(handler, options = {}) {
const workers = /* @__PURE__ */ new Map();
return this.queue.listen(async (message) => {
while (workers.size >= this.workers) {
const consumedId = await Promise.any(workers.values());
workers.delete(consumedId);
}
const workerId = crypto.randomUUID();
const promise = this.#work(workerId, handler, message);
workers.set(workerId, promise);
}, options);
}
async #work(workerId, handler, message) {
await this.#sleep(0);
await handler(message);
return workerId;
}
#sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
};
//#endregion
//#region federation/mq.test.ts
test("InProcessMessageQueue", async (t) => {
const mq = new InProcessMessageQueue();
await t.step("nativeRetrial property", () => {
assertFalse(mq.nativeRetrial);
});
const messages = [];
const controller = new AbortController();
const listening = mq.listen((message) => {
messages.push(message);
}, controller);
await t.step("enqueue()", async () => {
await mq.enqueue("Hello, world!");
});
await waitFor(() => messages.length > 0, 15e3);
await t.step("listen()", () => {
assertEquals(messages, ["Hello, world!"]);
});
let started = 0;
await t.step("enqueue() with delay", async () => {
started = Date.now();
await mq.enqueue("Delayed message", { delay: Temporal.Duration.from({ seconds: 3 }) });
assertEquals(messages, ["Hello, world!"]);
});
await waitFor(() => messages.length > 1, 15e3);
await t.step("listen() with delay", () => {
assertEquals(messages, ["Hello, world!", "Delayed message"]);
assertGreater(Date.now() - started, 3e3);
});
while (messages.length > 0) messages.pop();
await t.step("enqueueMany()", async () => {
const testMessages = Array.from({ length: 5 }, (_, i) => `Batch message ${i}!`);
await mq.enqueueMany(testMessages);
});
await waitFor(() => messages.length >= 5, 15e3);
await t.step("listen() [multiple]", () => {
assertEquals(messages.length, 5);
for (let i = 0; i < 5; i++) assertEquals(messages[i], `Batch message ${i}!`);
});
while (messages.length > 0) messages.pop();
started = 0;
await t.step("enqueueMany() with delay", async () => {
started = Date.now();
const testMessages = Array.from({ length: 3 }, (_, i) => `Delayed batch ${i}!`);
await mq.enqueueMany(testMessages, { delay: Temporal.Duration.from({ seconds: 2 }) });
assertEquals(messages.length, 0);
});
await waitFor(() => messages.length >= 3, 15e3);
await t.step("listen() [delayed multiple]", () => {
assertEquals(messages.length, 3);
assertGreater(Date.now() - started, 2e3);
for (let i = 0; i < 3; i++) assertEquals(messages[i], `Delayed batch ${i}!`);
});
controller.abort();
await listening;
});
test("MessageQueue.nativeRetrial", async (t) => {
if ("Deno" in globalThis && "openKv" in globalThis.Deno && typeof globalThis.Deno.openKv === "function") await t.step("DenoKvMessageQueue", async () => {
const { DenoKvMessageQueue } = await import("../denokv-CvROlGYX.js");
const mq = new DenoKvMessageQueue(await globalThis.Deno.openKv(":memory:"));
assert(mq.nativeRetrial);
if (Symbol.dispose in mq) {
const dispose = mq[Symbol.dispose];
if (typeof dispose === "function") dispose.call(mq);
}
});
await t.step("WorkersMessageQueue mock", () => {
class MockQueue {
send(_message, _options) {
return Promise.resolve();
}
sendBatch(_messages, _options) {
return Promise.resolve();
}
}
class TestWorkersMessageQueue {
nativeRetrial = true;
#queue;
constructor(queue) {
this.#queue = queue;
}
enqueue(message) {
return this.#queue.send(message);
}
enqueueMany(messages) {
return this.#queue.sendBatch(messages);
}
listen() {
throw new TypeError("WorkersMessageQueue does not support listen()");
}
}
const mq = new TestWorkersMessageQueue(new MockQueue());
assert(mq.nativeRetrial);
});
});
const queues = { InProcessMessageQueue: () => Promise.resolve(new InProcessMessageQueue()) };
if ("Deno" in globalThis && "openKv" in globalThis.Deno && typeof globalThis.Deno.openKv === "function") {
const { DenoKvMessageQueue } = await import("../denokv-CvROlGYX.js");
queues.DenoKvMessageQueue = async () => new DenoKvMessageQueue(await globalThis.Deno.openKv(":memory:"));
}
for (const mqName in queues) test({
name: `ParallelMessageQueue [${mqName}]`,
ignore: "Bun" in globalThis,
async fn(t) {
const mq = await queues[mqName]();
const workers = new ParallelMessageQueue(mq, 5);
await t.step("nativeRetrial property inheritance", () => {
assertEquals(workers.nativeRetrial, mq.nativeRetrial);
});
const messages = [];
const controller = new AbortController();
const listening = workers.listen(async (message) => {
for (let i = 0, cnt = 5 + Math.random() * 5; i < cnt; i++) await delay(250);
messages.push(message);
}, controller);
await t.step("enqueue() [single]", async () => {
await workers.enqueue("Hello, world!");
});
await waitFor(() => messages.length > 0, 15e3);
await t.step("listen() [single]", () => {
assertEquals(messages, ["Hello, world!"]);
});
messages.pop();
await t.step("enqueue() [multiple]", async () => {
for (let i = 0; i < 20; i++) await workers.enqueue(`Hello, ${i}!`);
});
await t.step("listen() [multiple]", async () => {
await delay(3e3);
assertGreaterOrEqual(messages.length, 5);
await waitFor(() => messages.length >= 20, 15e3);
assertEquals(messages.length, 20);
});
await waitFor(() => messages.length >= 20, 15e3);
while (messages.length > 0) messages.pop();
await t.step("enqueueMany()", async () => {
const messages$1 = Array.from({ length: 20 }, (_, i) => `Hello, ${i}!`);
await workers.enqueueMany(messages$1);
});
await t.step("listen() [multiple]", async () => {
await delay(3e3);
assertGreaterOrEqual(messages.length, 5);
await waitFor(() => messages.length >= 20, 15e3);
assertEquals(messages.length, 20);
});
await waitFor(() => messages.length >= 20, 15e3);
controller.abort();
await listening;
if (Symbol.dispose in mq) {
const dispose = mq[Symbol.dispose];
if (typeof dispose === "function") dispose.call(mq);
}
}
});
async function waitFor(predicate, timeoutMs) {
const started = Date.now();
while (!predicate()) {
await delay(500);
if (Date.now() - started > timeoutMs) throw new Error("Timeout");
}
}
//#endregion