@miniflare/queues
Version:
Workers Queues module for Miniflare: a fun, full-featured, fully-local simulator for Cloudflare Workers
357 lines (353 loc) • 11 kB
JavaScript
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __markAsModule = (target) => __defProp(target, "__esModule", { value: true });
var __export = (target, all) => {
__markAsModule(target);
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __reExport = (target, module2, desc) => {
if (module2 && typeof module2 === "object" || typeof module2 === "function") {
for (let key of __getOwnPropNames(module2))
if (!__hasOwnProp.call(target, key) && key !== "default")
__defProp(target, key, { get: () => module2[key], enumerable: !(desc = __getOwnPropDesc(module2, key)) || desc.enumerable });
}
return target;
};
var __toModule = (module2) => {
return __reExport(__markAsModule(__defProp(module2 != null ? __create(__getProtoOf(module2)) : {}, "default", module2 && module2.__esModule && "default" in module2 ? { get: () => module2.default, enumerable: true } : { value: module2, enumerable: true })), module2);
};
var __decorateClass = (decorators, target, key, kind) => {
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
for (var i = decorators.length - 1, decorator; i >= 0; i--)
if (decorator = decorators[i])
result = (kind ? decorator(target, key, result) : decorator(result)) || result;
if (kind && result)
__defProp(target, key, result);
return result;
};
// packages/queues/src/index.ts
__export(exports, {
DEFAULT_BATCH_SIZE: () => DEFAULT_BATCH_SIZE,
DEFAULT_RETRIES: () => DEFAULT_RETRIES,
DEFAULT_WAIT_MS: () => DEFAULT_WAIT_MS,
Message: () => Message,
MessageBatch: () => MessageBatch,
QueueBroker: () => QueueBroker,
QueueError: () => QueueError,
QueuesPlugin: () => QueuesPlugin,
WorkerQueue: () => WorkerQueue,
kSetFlushCallback: () => kSetFlushCallback
});
// packages/queues/src/plugin.ts
var import_shared = __toModule(require("@miniflare/shared"));
var DEFAULT_BATCH_SIZE = 5;
var DEFAULT_WAIT_MS = 1e3;
var DEFAULT_RETRIES = 2;
var QueuesPlugin = class extends import_shared.Plugin {
queueBindings;
queueConsumers;
#consumers;
constructor(ctx, options) {
super(ctx);
this.assignOptions(options);
if (options?.queueBindings?.length || options?.queueConsumers?.length) {
ctx.log.warn("Queues are experimental. There may be breaking changes in the future.");
}
this.#consumers = (this.queueConsumers ?? []).map((entry) => {
let opts;
if (typeof entry === "string") {
opts = {
queueName: entry
};
} else {
opts = entry;
}
return {
queueName: opts.queueName,
maxBatchSize: opts.maxBatchSize ?? DEFAULT_BATCH_SIZE,
maxWaitMs: opts.maxWaitMs ?? DEFAULT_WAIT_MS,
maxRetries: opts.maxRetries ?? DEFAULT_RETRIES,
deadLetterQueue: opts.deadLetterQueue,
dispatcher: this.ctx.queueEventDispatcher
};
});
}
async setup(_storageFactory) {
const bindings = {};
for (const binding of this.queueBindings ?? []) {
bindings[binding.name] = this.ctx.queueBroker.getOrCreateQueue(binding.queueName);
}
const requiresModuleExports = this.#consumers.length > 0;
return { bindings, requiresModuleExports };
}
beforeReload() {
for (const consumer of this.#consumers) {
const queue = this.ctx.queueBroker.getOrCreateQueue(consumer.queueName);
this.ctx.queueBroker.setConsumer(queue, consumer);
}
}
};
__decorateClass([
(0, import_shared.Option)({
type: import_shared.OptionType.OBJECT,
name: "queue",
alias: "q",
description: "Queue Bindings",
logName: "Queue Bindings",
typeFormat: "NAME=QUEUE",
logValue: (bindings) => bindings.map((b) => b.name).join(", "),
fromEntries: (entries) => entries.map(([k, v]) => {
return { name: k, queueName: v };
}),
fromWrangler: (wranglerConfig) => wranglerConfig.queues?.producers?.map((b) => {
return { name: b.binding, queueName: b.queue };
})
})
], QueuesPlugin.prototype, "queueBindings", 2);
__decorateClass([
(0, import_shared.Option)({
type: import_shared.OptionType.ARRAY,
name: "queue-consumers",
description: "Queue Consumers",
logName: "Queue Consumers",
logValue: (consumers) => consumers.map((b) => b.queueName).join(", "),
fromWrangler: (wranglerConfig) => wranglerConfig.queues?.consumers?.map((opts) => {
const result = { queueName: opts.queue };
if (opts.batch_size) {
result.maxBatchSize = opts.batch_size;
}
if (opts.batch_timeout) {
result.maxWaitMs = 1e3 * opts.batch_timeout;
}
if (opts.message_retries) {
result.maxRetries = opts.message_retries;
}
if (opts.dead_letter_queue) {
result.deadLetterQueue = opts.dead_letter_queue;
}
return result;
})
})
], QueuesPlugin.prototype, "queueConsumers", 2);
// packages/queues/src/broker.ts
var import_shared2 = __toModule(require("@miniflare/shared"));
var QueueError = class extends import_shared2.MiniflareError {
};
var kGetPendingRetry = Symbol("kGetPendingRetry");
var kPrepareForRetry = Symbol("kPrepareForRetry");
var kGetFailedAttempts = Symbol("kGetFailedAttempts");
var Message = class {
constructor(id, timestamp, body, log) {
this.id = id;
this.timestamp = timestamp;
this.body = (globalThis.structuredClone ?? import_shared2.structuredCloneImpl)(body);
this.#log = log;
this.#pendingRetry = false;
this.#failedAttempts = 0;
}
body;
#log;
#pendingRetry;
#failedAttempts;
retry() {
this.#pendingRetry = true;
}
[kPrepareForRetry]() {
this.#pendingRetry = false;
this.#failedAttempts++;
}
[kGetPendingRetry]() {
return this.#pendingRetry;
}
[kGetFailedAttempts]() {
return this.#failedAttempts;
}
};
var MessageBatch = class {
queue;
messages;
constructor(queue, messages) {
this.queue = queue;
this.messages = messages;
}
retryAll() {
for (const msg of this.messages) {
msg.retry();
}
}
};
var FlushType;
(function(FlushType2) {
FlushType2[FlushType2["NONE"] = 0] = "NONE";
FlushType2[FlushType2["DELAYED"] = 1] = "DELAYED";
FlushType2[FlushType2["IMMEDIATE"] = 2] = "IMMEDIATE";
})(FlushType || (FlushType = {}));
var kSetFlushCallback = Symbol("kSetFlushCallback");
var WorkerQueue = class {
#broker;
#queueName;
#log;
#consumer;
#messages;
#messageCounter;
#pendingFlush;
#timeout;
#flushCallback;
constructor(broker, queueName, log) {
this.#broker = broker;
this.#queueName = queueName;
this.#log = log;
this.#messages = [];
this.#messageCounter = 0;
this.#pendingFlush = 0;
}
async send(body, options) {
this.#enqueue(body, options);
}
async sendBatch(batch) {
for (const req of batch) {
this.#enqueue(req.body, req);
}
}
[import_shared2.kSetConsumer](consumer) {
if (consumer === void 0) {
clearTimeout(this.#timeout);
this.#pendingFlush = 0;
this.#consumer = void 0;
return;
}
if (this.#consumer) {
throw new QueueError("ERR_CONSUMER_ALREADY_SET");
}
this.#consumer = consumer;
if (this.#messages.length) {
this.#ensurePendingFlush();
}
}
[import_shared2.kGetConsumer]() {
return this.#consumer ?? null;
}
#enqueue(body, _options) {
const msg = new Message(`${this.#queueName}-${this.#messageCounter}`, new Date(), body, this.#log);
this.#messages.push(msg);
this.#messageCounter++;
if (this.#consumer) {
this.#ensurePendingFlush();
}
}
#ensurePendingFlush() {
if (!this.#consumer) {
return;
}
if (this.#pendingFlush === 2) {
return;
}
if (this.#pendingFlush === 1) {
if (this.#messages.length < this.#consumer?.maxBatchSize) {
return;
}
clearTimeout(this.#timeout);
this.#timeout = void 0;
}
const newFlushType = this.#messages.length < this.#consumer.maxBatchSize ? 1 : 2;
this.#pendingFlush = newFlushType;
const delay = newFlushType === 1 ? this.#consumer?.maxWaitMs : 0;
this.#timeout = setTimeout(() => {
this.#flush();
if (this.#flushCallback) {
this.#flushCallback();
}
}, delay);
}
async #flush() {
if (!this.#consumer) {
return;
}
const maxAttempts = this.#consumer.maxRetries + 1;
const deadLetterQueueName = this.#consumer.deadLetterQueue;
const msgs = this.#messages.slice(0, this.#consumer.maxBatchSize);
const batch = new MessageBatch(this.#queueName, msgs);
this.#messages = this.#messages.slice(msgs.length);
try {
await this.#consumer?.dispatcher(batch);
} catch (err) {
this.#log?.error((0, import_shared2.prefixError)(`${this.#queueName} Consumer`, err));
batch.retryAll();
}
this.#pendingFlush = 0;
this.#timeout = void 0;
const toRetry = [];
const toDLQ = [];
batch.messages.forEach((msg) => {
if (!msg[kGetPendingRetry]()) {
return;
}
msg[kPrepareForRetry]();
if (msg[kGetFailedAttempts]() < maxAttempts) {
this.#log?.debug(`Retrying message "${msg.id}"...`);
toRetry.push(msg);
} else if (deadLetterQueueName) {
this.#log?.warn(`Moving message "${msg.id}" to dead letter queue "${deadLetterQueueName}"...`);
toDLQ.push(msg);
} else {
this.#log?.warn(`Dropped message "${msg.id}" after ${maxAttempts} failed attempts!`);
}
});
if (toRetry.length) {
this.#messages.push(...toRetry);
}
if (this.#messages.length > 0) {
this.#ensurePendingFlush();
}
if (deadLetterQueueName) {
const deadLetterQueue = this.#broker.getOrCreateQueue(deadLetterQueueName);
toDLQ.forEach((msg) => {
deadLetterQueue.send(msg.body);
});
}
}
[kSetFlushCallback](callback) {
this.#flushCallback = callback;
}
};
var QueueBroker = class {
#queues;
#log;
constructor(log) {
this.#queues = new Map();
this.#log = log;
}
getOrCreateQueue(name) {
let queue = this.#queues.get(name);
if (queue === void 0) {
this.#queues.set(name, queue = new WorkerQueue(this, name, this.#log));
}
return queue;
}
resetConsumers() {
for (const queue of this.#queues.values())
queue[import_shared2.kSetConsumer]();
}
setConsumer(queue, consumer) {
queue[import_shared2.kSetConsumer](consumer);
}
};
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
DEFAULT_BATCH_SIZE,
DEFAULT_RETRIES,
DEFAULT_WAIT_MS,
Message,
MessageBatch,
QueueBroker,
QueueError,
QueuesPlugin,
WorkerQueue,
kSetFlushCallback
});
//# sourceMappingURL=index.js.map