alepha
Version:
Easy-to-use modern TypeScript framework for building many kind of applications.
447 lines (446 loc) • 14.2 kB
JavaScript
import { $atom, $hook, $inject, $module, $state, Alepha, AlephaError, KIND, PipelinePrimitive, Primitive, createPrimitive, z } from "alepha";
import { $logger } from "alepha/logger";
import { DateTimeProvider } from "alepha/datetime";
//#region ../../src/queue/core/primitives/$consumer.ts
/**
* Creates a consumer primitive to process messages from a specific queue.
*
* Provides a dedicated message consumer that connects to a queue and processes messages
* with custom handler logic, enabling scalable architectures where multiple consumers
* can process messages from the same queue.
*
* **Key Features**
* - Seamless integration with any $queue primitive
* - Full type safety inherited from queue schema
* - Automatic worker management for background processing
* - Built-in error handling and retry mechanisms
* - Support for multiple consumers per queue for horizontal scaling
*
* **Common Use Cases**
* - Email sending and notification services
* - Image and media processing workers
* - Data synchronization and background jobs
*
* @example
* ```ts
* class EmailService {
* emailQueue = $queue({
* name: "emails",
* schema: z.object({
* to: z.text(),
* subject: z.text(),
* body: z.text()
* })
* });
*
* emailConsumer = $consumer({
* queue: this.emailQueue,
* handler: async (message) => {
* const { to, subject, body } = message.payload;
* await this.sendEmail(to, subject, body);
* }
* });
*
* async sendWelcomeEmail(userEmail: string) {
* await this.emailQueue.push({
* to: userEmail,
* subject: "Welcome!",
* body: "Thanks for joining."
* });
* }
* }
* ```
*/
const $consumer = (options) => {
return createPrimitive(ConsumerPrimitive, options);
};
var ConsumerPrimitive = class extends PipelinePrimitive {};
$consumer[KIND] = ConsumerPrimitive;
//#endregion
//#region ../../src/queue/core/providers/MemoryQueueProvider.ts
var MemoryQueueProvider = class {
log = $logger();
queueList = {};
async push(queue, ...messages) {
if (this.queueList[queue] == null) this.queueList[queue] = [];
this.queueList[queue].push(...messages);
}
async pop(queue) {
return this.queueList[queue]?.shift();
}
};
//#endregion
//#region ../../src/queue/core/providers/QueueProvider.ts
/**
* Minimalist Queue interface.
*
* Will be probably enhanced in the future to support more advanced features. But for now, it's enough!
*/
var QueueProvider = class {};
//#endregion
//#region ../../src/queue/core/providers/WorkerProvider.ts
/**
* Queue worker configuration atom.
*/
const queueWorkerOptions = $atom({
name: "alepha.queue.worker.options",
schema: z.object({
interval: z.integer().describe("Interval in milliseconds to wait before checking for new messages.").default(1e3),
maxInterval: z.integer().describe("Maximum interval in milliseconds to wait before checking for new messages.").default(32e3),
concurrency: z.integer().describe("Number of workers to run concurrently. Useful only if you are doing a lot of I/O.").default(1)
}),
default: {
interval: 1e3,
maxInterval: 32e3,
concurrency: 1
}
});
var WorkerProvider = class {
log = $logger();
options = $state(queueWorkerOptions);
alepha = $inject(Alepha);
queueProvider = $inject(QueueProvider);
dateTimeProvider = $inject(DateTimeProvider);
workerPromises = [];
workersRunning = 0;
abortController;
workerIntervals = {};
consumers = [];
nextConsumerIndex = 0;
get isRunning() {
return this.workersRunning > 0;
}
start = $hook({
on: "start",
priority: "last",
handler: () => {
for (const queue of this.alepha.primitives($queue)) {
const handler = queue.options.handler;
if (handler) this.consumers.push({
handler,
queue
});
}
for (const consumer of this.alepha.primitives($consumer)) this.consumers.push({
queue: consumer.options.queue,
handler: (msg) => consumer.handler.run(msg)
});
if (this.consumers.length > 0) {
this.startWorkers();
this.log.debug(`Watching for ${this.consumers.length} queue${this.consumers.length > 1 ? "s" : ""} with ${this.options.concurrency} worker${this.options.concurrency > 1 ? "s" : ""}.`);
}
}
});
/**
* Start the workers.
* This method will create an endless loop that will check for new messages!
*/
startWorkers() {
this.abortController ??= new AbortController();
const workerToStart = this.options.concurrency - this.workersRunning;
for (let i = 0; i < workerToStart; i++) {
this.workersRunning += 1;
this.log.debug(`Starting worker n-${i}`);
const workerLoop = async () => {
while (this.workersRunning > 0) {
this.log.trace(`Worker n-${i} is checking for new messages`);
const next = await this.getNextMessage();
if (next) {
this.workerIntervals[i] = 0;
await this.processMessage(next);
} else await this.waitForNextMessage(i);
}
this.log.info(`Worker n-${i} has stopped`);
if (this.workersRunning > 0) this.workersRunning -= 1;
};
this.workerPromises.push(workerLoop().catch((e) => {
this.log.error(`Worker n-${i} has crashed`, e);
this.workersRunning = Math.max(0, this.workersRunning - 1);
}));
}
}
stop = $hook({
on: "stop",
handler: async () => {
if (this.consumers.length > 0) await this.stopWorkers();
}
});
/**
* Wait for the next message, where `n` is the worker number.
*
* This method will wait for a certain amount of time, increasing the wait time again if no message is found.
*/
async waitForNextMessage(n) {
const intervals = this.workerIntervals;
const milliseconds = intervals[n] || this.options.interval;
this.log.trace(`Worker n-${n} is waiting for ${milliseconds}ms.`);
if (this.abortController?.signal.aborted) {
this.log.warn(`Worker n-${n} aborted.`);
return;
}
await this.dateTimeProvider.wait(milliseconds, { signal: this.abortController?.signal });
if (intervals[n]) {
if (intervals[n] < this.options.maxInterval) intervals[n] = intervals[n] * 2;
} else intervals[n] = milliseconds;
}
/**
* Get the next message.
*/
async getNextMessage() {
const len = this.consumers.length;
for (let i = 0; i < len; i++) {
const idx = (this.nextConsumerIndex + i) % len;
const consumer = this.consumers[idx];
const message = await consumer.queue.provider.pop(consumer.queue.name);
if (message) {
this.nextConsumerIndex = (idx + 1) % len;
return {
message,
consumer
};
}
}
}
/**
* Process a message from a queue.
*/
async processMessage(response) {
const { message, consumer } = response;
try {
const json = JSON.parse(message);
const payload = this.alepha.codec.decode(consumer.queue.options.schema, json.payload);
await this.alepha.context.run(() => consumer.handler({ payload }));
} catch (e) {
this.log.error("Failed to process message", e);
}
}
/**
* Stop the workers.
*
* This method will stop the workers and wait for them to finish processing.
*/
async stopWorkers() {
this.workersRunning = 0;
this.log.trace("Stopping workers...");
this.abortController?.abort();
this.log.trace("Waiting for workers to finish...");
await Promise.all(this.workerPromises);
}
/**
* Force the workers to get back to work.
*/
wakeUp() {
this.log.debug("Waking up workers...");
this.abortController?.abort();
this.abortController = new AbortController();
this.startWorkers();
}
};
//#endregion
//#region ../../src/queue/core/primitives/$queue.ts
/**
* Creates a queue primitive for asynchronous message processing with background workers.
*
* `$queue` is the **raw transport layer**: it fans messages out to background
* workers over the configured backend with type-safe payloads. Delivery is
* **at-most-once** — a message is popped from the backend before the handler
* runs, so a handler error or a process crash loses it. There is no retry,
* no dead-letter queue, and no delivery guarantee at this layer.
*
* **For work that must not be lost, use `$job` (alepha/api/jobs) instead.**
* It layers a durable, DB-backed outbox over this transport: at-least-once
* delivery, retries, idempotency keys, priorities, crash recovery via a
* reconciliation sweep, and failure records.
*
* **What $queue gives you**
* - Type-safe payloads with schema validation at push and receive
* - Background workers with graceful shutdown and lifecycle management
* - Pluggable backends: memory (dev/test), Redis, Cloudflare Queues
* - Cheap fire-and-forget fan-out where occasional loss is acceptable
* (cache invalidation, presence pings, metrics, live notifications)
*
* @example Loss-tolerant event fan-out
* ```typescript
* const activityQueue = $queue({
* name: "activity-events",
* schema: z.object({
* userId: z.text(),
* event: z.text(),
* at: z.number()
* }),
* handler: async (message) => {
* await metrics.track(message.payload);
* }
* });
*
* // Push messages for background processing
* await activityQueue.push({
* userId: "u1",
* event: "page-view",
* at: 1700000000000
* });
* ```
*
* @example Batch processing with Redis
* ```typescript
* const imageQueue = $queue({
* name: "image-processing",
* provider: RedisQueueProvider,
* schema: z.object({
* imageId: z.text(),
* operations: z.array(z.enum(["resize", "compress", "thumbnail"]))
* }),
* handler: async (message) => {
* for (const op of message.payload.operations) {
* await processImage(message.payload.imageId, op);
* }
* }
* });
*
* // Batch processing multiple images
* await imageQueue.push(
* { imageId: "img1", operations: ["resize", "thumbnail"] },
* { imageId: "img2", operations: ["compress"] },
* { imageId: "img3", operations: ["resize", "compress", "thumbnail"] }
* );
* ```
*
* @example Development with memory provider
* ```typescript
* const taskQueue = $queue({
* name: "dev-tasks",
* provider: "memory",
* schema: z.object({
* taskType: z.enum(["cleanup", "backup", "report"]),
* data: z.record(z.text(), z.any())
* }),
* handler: async (message) => {
* switch (message.payload.taskType) {
* case "cleanup":
* await performCleanup(message.payload.data);
* break;
* case "backup":
* await createBackup(message.payload.data);
* break;
* case "report":
* await generateReport(message.payload.data);
* break;
* }
* }
* });
* ```
*/
const $queue = (options) => {
return createPrimitive(QueuePrimitive, options);
};
var QueuePrimitive = class extends Primitive {
log = $logger();
workerProvider = $inject(WorkerProvider);
provider = this.$provider();
async push(...payloads) {
await Promise.all(payloads.map((payload) => this.provider.push(this.name, JSON.stringify({
headers: {},
payload: this.alepha.codec.decode(this.options.schema, payload)
}))));
this.log.debug(`Pushed to queue ${this.name}`, payloads);
this.workerProvider.wakeUp();
}
get name() {
return this.options.name || this.config.propertyKey;
}
$provider() {
if (!this.options.provider) return this.alepha.inject(QueueProvider);
if (this.options.provider === "memory") return this.alepha.inject(MemoryQueueProvider);
return this.alepha.inject(this.options.provider);
}
};
$queue[KIND] = QueuePrimitive;
//#endregion
//#region ../../src/queue/core/providers/CloudflareQueueProvider.ts
/**
* Default queue binding name used in wrangler configuration.
*/
const QUEUE_DEFAULT_BINDING = "JOBS_QUEUE";
/**
* How many times Cloudflare redelivers a message before routing it to the
* consumer's dead-letter queue. Matches Cloudflare's own default.
*/
const QUEUE_DEFAULT_MAX_RETRIES = 3;
/**
* Cloudflare Queue provider.
*
* Uses a Queue binding for message dispatch. Messages are wrapped with the
* logical queue name so the consumer can route them to the correct handler.
*
* **Required Cloudflare binding:**
* - `JOBS_QUEUE` - A Queue binding in wrangler configuration
*
* @example
* ```toml
* # wrangler.toml - automatically generated by alepha build
* [[queues.producers]]
* binding = "JOBS_QUEUE"
* queue = "my-app-queue"
* ```
*/
var CloudflareQueueProvider = class extends QueueProvider {
alepha = $inject(Alepha);
log = $logger();
queue;
onStart = $hook({
on: "start",
handler: async () => {
const cloudflareEnv = this.alepha.store.get("cloudflare.env");
if (!cloudflareEnv) throw new AlephaError("Cloudflare Workers environment not found in Alepha store under 'cloudflare.env'.");
const binding = cloudflareEnv[QUEUE_DEFAULT_BINDING];
if (!binding) throw new AlephaError(`Queue binding '${QUEUE_DEFAULT_BINDING}' not found in Cloudflare Workers environment.`);
this.queue = binding;
this.log.info("Cloudflare Queue ready");
}
});
async push(queue, message) {
await this.getQueue().send({
queue,
message
});
}
/**
* Not used on Cloudflare — queue consumption is push-based via the `queue` handler.
*/
async pop(_queue) {}
getQueue() {
if (!this.queue) throw new AlephaError("Queue binding not initialized. Call start() first.");
return this.queue;
}
};
//#endregion
//#region ../../src/queue/core/index.ts
/**
* Asynchronous message processing with automatic worker management.
*
* **Features:**
* - Background job queues with type-safe payloads
* - Queue consumer handlers
* - Automatic worker threads for non-blocking processing
* - Retry mechanisms with exponential backoff
* - Dead letter queues for failed messages
* - Batch processing support
* - Configurable concurrency and worker pools
* - Providers: Memory (dev), Redis (production)
*
* @module alepha.queue
*/
const AlephaQueue = $module({
name: "alepha.queue",
primitives: [$queue, $consumer],
services: [QueueProvider, WorkerProvider],
variants: [MemoryQueueProvider],
register: (alepha) => alepha.with({
optional: true,
provide: QueueProvider,
use: MemoryQueueProvider
}).with(WorkerProvider)
});
//#endregion
export { $consumer, $queue, AlephaQueue, CloudflareQueueProvider, ConsumerPrimitive, MemoryQueueProvider, QUEUE_DEFAULT_BINDING, QUEUE_DEFAULT_MAX_RETRIES, QueuePrimitive, QueueProvider, WorkerProvider, queueWorkerOptions };
//# sourceMappingURL=index.js.map