@midwayjs/bullmq
Version:
midway component for BullMQ
281 lines • 10.8 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BullMQFramework = exports.BullMQQueue = void 0;
const core_1 = require("@midwayjs/core");
const bullmq_1 = require("bullmq");
const constants_1 = require("./constants");
class BullMQQueue extends bullmq_1.Queue {
constructor(queueName, queueOptions) {
super(queueName, queueOptions);
this.queueName = queueName;
this.queueOptions = queueOptions;
this.queueEventsList = [];
this.queueEventsProducerList = [];
}
async addJobToQueue(data, options) {
const { repeat, ...OtherOptions } = options !== null && options !== void 0 ? options : {};
if (repeat) {
return this.upsertJobScheduler(this.name, repeat, {
name: 'jobName',
data,
opts: OtherOptions, // additional job options
});
}
return this.add('jobName', data || {}, options);
}
/**
* @deprecated use addJobToQueue instead
*/
// runJob 与 @midwayjs/bull 保持一致，如果想要使用 jobName 则可以直接调用 queue.add
async runJob(data, options) {
return this.addJobToQueue(data, options);
}
getQueueName() {
return this.queueName;
}
createQueueEvents(options) {
const evt = new bullmq_1.QueueEvents(this.name, {
connection: this.queueOptions.connection,
prefix: this.queueOptions.prefix,
...options,
});
this.queueEventsList.push(evt);
return evt;
}
getQueueEventsList() {
return this.queueEventsList;
}
createQueueEventsProducer(options) {
const producer = new bullmq_1.QueueEventsProducer(this.name, {
connection: this.queueOptions.connection,
prefix: this.queueOptions.prefix,
...options,
});
this.queueEventsProducerList.push(producer);
return producer;
}
getQueueEventsProducerList() {
return this.queueEventsProducerList;
}
async close() {
// 并发关闭
await Promise.all(this.queueEventsList.map(evt => evt.close()));
await Promise.all(this.queueEventsProducerList.map(producer => producer.close()));
await super.close();
}
}
exports.BullMQQueue = BullMQQueue;
let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
constructor() {
super(...arguments);
this.queueMap = new Map();
this.workerMap = new Map();
this.flowProducerMap = new Map();
}
async applicationInitialize(options) {
this.app = {};
}
loadConfig() {
const defaultConnection = this.configService.getConfiguration('bullmq.defaultConnection');
const defaultPrefix = this.configService.getConfiguration('bullmq.defaultPrefix');
this.defaultConnection = {
connection: defaultConnection,
prefix: defaultPrefix,
};
this.defaultQueueConfig = this.configService.getConfiguration('bullmq.defaultQueueOptions');
this.defaultWorkerConfig = this.configService.getConfiguration('bullmq.defaultWorkerOptions');
this.clearRepeatJobWhenStart = this.configService.getConfiguration('bullmq.clearRepeatJobWhenStart');
}
configure() {
return this.configService.getConfiguration('bullmq');
}
getFrameworkName() {
return 'bullmq';
}
async run() {
var _a, _b, _c;
const processorModules = (0, core_1.listModule)(constants_1.BULLMQ_PROCESSOR_KEY);
for (const mod of processorModules) {
const options = (0, core_1.getClassMetadata)(constants_1.BULLMQ_PROCESSOR_KEY, mod);
const { repeat, ...otherOptions } = (_a = options.jobOptions) !== null && _a !== void 0 ? _a : {};
const queueOptions = (_b = options.queueOptions) !== null && _b !== void 0 ? _b : {};
const currentQueue = this.ensureQueue(options.queueName, {
...queueOptions,
defaultJobOptions: otherOptions,
});
if (!currentQueue) {
throw new core_1.MidwayCommonError(`[midway:bullmq] Queue ${options.queueName} not found`);
}
// clear old repeat job when start
if (this.clearRepeatJobWhenStart) {
const jobs = await currentQueue.getJobSchedulers();
for (const job of jobs) {
await currentQueue.removeJobScheduler(job.key);
}
// Repeatable in jobOptions is depecrate
const repeatableJobs = await currentQueue.getRepeatableJobs();
for (const job of repeatableJobs) {
await currentQueue.removeRepeatableByKey(job.key);
}
}
await this.addProcessor(mod, options.queueName, options.workerOptions);
if (repeat) {
// add repeatable job
await ((_c = this.getQueue(options.queueName)) === null || _c === void 0 ? void 0 : _c.addJobToQueue({}, options.jobOptions));
}
}
}
async beforeStop() {
// loop queueMap and stop all queue
await Promise.all(Array.from(this.queueMap.values()).map(queue => queue.close()));
await Promise.all(Array.from(this.workerMap.values()).map(worker => worker.map(w => w.close())));
await Promise.all(Array.from(this.flowProducerMap.values()).map(producer => producer.close()));
}
/**
* Create a queue with name and queueOptions
*/
createQueue(name, queueOptions = {}) {
const mergedOptions = (0, core_1.extend)(true, {}, this.defaultQueueConfig, this.defaultConnection, queueOptions);
const queue = new BullMQQueue(name, mergedOptions);
this.queueMap.set(name, queue);
queue.on('error', err => {
this.bullMQLogger.error(err);
});
return queue;
}
/**
* Get a queue by name
*/
getQueue(name) {
return this.queueMap.get(name);
}
/**
* Ensure a queue by name and queueOptions
*/
ensureQueue(name, queueOptions = {}) {
if (!this.queueMap.has(name)) {
this.createQueue(name, queueOptions);
}
return this.queueMap.get(name);
}
getQueueList() {
return Array.from(this.queueMap.values());
}
/**
* Get the first worker by queueName
*/
getWorker(queueName) {
var _a;
return (_a = this.workerMap.get(queueName)) === null || _a === void 0 ? void 0 : _a[0];
}
/**
* Get all workers by queueName
*/
getWorkers(queueName) {
return this.workerMap.get(queueName);
}
/**
* Create a worker
*/
createWorker(queueName, processor, workerOptions = {}) {
const merged = {
...this.defaultConnection,
...this.defaultWorkerConfig,
...workerOptions,
};
const worker = new bullmq_1.Worker(queueName, processor, merged);
if (!this.workerMap.has(queueName)) {
this.workerMap.set(queueName, []);
}
this.workerMap.get(queueName).push(worker);
return worker;
}
/**
* Add a processor class and init a worker
*/
async addProcessor(processor, queueName, workerOptions) {
const queue = this.queueMap.get(queueName);
if (!queue)
throw Error(`queue not found ${queueName}`);
return this.createWorker(queueName, async (job, token) => {
const ctx = this.app.createAnonymousContext({
jobId: job.id,
job,
token,
from: processor,
});
try {
ctx.logger.info(`start process job ${job.id} from ${processor.name}`);
const isPassed = await this.app
.getFramework()
.runGuard(ctx, processor, 'execute');
if (!isPassed) {
throw new core_1.MidwayInvokeForbiddenError('execute', processor);
}
const service = await ctx.requestContext.getAsync(processor);
const fn = await this.applyMiddleware(async (ctx) => {
return await core_1.Utils.toAsyncFunction(service.execute.bind(service))(job.data, job, token);
});
const result = await Promise.resolve(await fn(ctx));
ctx.logger.info(`complete process job ${job.id} from ${processor.name}`);
return result;
}
catch (err) {
ctx.logger.error(err);
return Promise.reject(err);
}
}, workerOptions);
}
/**
* Add a job to the queue
*/
async addJobToQueue(queueName, jobData, options) {
const queue = this.queueMap.get(queueName);
if (queue) {
return await queue.addJobToQueue(jobData, options);
}
}
/**
* @deprecated use addJobToQueue instead
*/
async runJob(queueName, jobData, options) {
return this.addJobToQueue(queueName, jobData, options);
}
/**
* Create a flow producer, if producerName is provided, it will be store.
*/
createFlowProducer(options, producerName) {
const producer = new bullmq_1.FlowProducer({
...this.defaultConnection,
...options,
});
if (producerName) {
this.flowProducerMap.set(producerName, producer);
}
return producer;
}
/**
* Get a flow producer by name
*/
getFlowProducer(producerName) {
return this.flowProducerMap.get(producerName);
}
};
__decorate([
(0, core_1.Logger)('bullMQLogger'),
__metadata("design:type", Object)
], BullMQFramework.prototype, "bullMQLogger", void 0);
BullMQFramework = __decorate([
(0, core_1.Framework)()
], BullMQFramework);
exports.BullMQFramework = BullMQFramework;
//# sourceMappingURL=framework.js.map