pdmq
Version:
314 lines (313 loc) • 13.9 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.PDMQConsumer = void 0;
const uuid_1 = require("uuid");
const task_service_1 = require("./../services/main/task.service");
const redis_1 = require("redis");
const node_cron_1 = require("node-cron");
const cron_expression_enum_1 = require("../enums/cron-expression.enum");
const moment = require("moment");
const rxjs_1 = require("rxjs");
const redis_service_1 = require("../services/common/redis.service");
const task_trigger_types_enum_1 = require("../enums/task-trigger-types.enum");
const os = require("os");
const processing_instant_task_key_const_1 = require("../constants/processing-instant-task-key.const");
class PDMQConsumer {
constructor(InitialOptions) {
this.InitialOptions = InitialOptions;
this.taskQueue = new rxjs_1.Subject();
this.selectedObserver = 0;
this.instanceId = uuid_1.v1();
this.init();
this.consumerIdentity = InitialOptions.consumer_identity || this.instanceId;
}
/**
* Initial Client and Consumer
*/
init() {
if (this.InitialOptions.debug)
console.debug("Initial Consumer");
this.redisClient = redis_1.createClient({
url: this.InitialOptions.redis_url
});
this.redisClient.on("ready", () => __awaiter(this, void 0, void 0, function* () {
this.redisService = new redis_service_1.RedisService(this.redisClient);
this.taskService = new task_service_1.TaskService();
/** Lookup any expired time based task */
yield this.lookupExpiredTasks();
/** Clean consumer stats */
yield this.clean();
node_cron_1.schedule(cron_expression_enum_1.CronExpression.EVERY_SECOND, () => this.lookup());
}));
}
/**
* Disconnect Redis
*
* @returns {void}
*/
disconnect() {
return this.redisClient.quit();
}
/**
* Rotate Expired Tasks
*
* @returns {void}
*/
lookupExpiredTasks() {
return __awaiter(this, void 0, void 0, function* () {
/** List all tasks */
const keys = yield this.redisService.searchKeys("tasks:*");
/** tasks:2021:03:19:20:00:00:1c8cfcf0-885d-11eb-b54d-bf0a5991c053 */
const expiredKeys = keys.filter(key => {
const arr = key.split(":");
if (arr.length != 8)
return false;
return moment(arr.slice(1, 7).join(":"), 'YYYY:MM:DD:HH:mm:ss', true).isBefore(moment());
});
if (!expiredKeys.length)
return;
if (this.InitialOptions.debug)
console.debug("Recreate expired tasks: ", expiredKeys);
const taskStrings = yield this.redisService.getMulti(expiredKeys);
for (const taskString of taskStrings) {
const task = JSON.parse(taskString);
yield this.taskService.addTask(task, this.redisService);
}
;
yield this.redisService.delete(expiredKeys);
});
}
/**
* Delete All Tasks
*/
clean() {
return __awaiter(this, void 0, void 0, function* () {
const keys = yield this.redisService.searchKeys("consumer:*:count");
yield this.redisService.delete(keys);
});
}
/**
* Rotate Task
*
* @param task
* @returns {PDMQTask}
*/
rotateTask(task) {
return __awaiter(this, void 0, void 0, function* () {
if (task.task_trigger_type == task_trigger_types_enum_1.TaskTriggerTypes.ONCE)
return;
const nextRunTime = this.taskService.getNextRunTime(task);
const momentNextRunTime = moment(nextRunTime, 'YYYY:MM:DD:HH:mm:ss', true);
task.trigger_next_at = momentNextRunTime.toISOString();
if (momentNextRunTime.isValid() && momentNextRunTime.isAfter(moment())) {
this.redisService.set(`tasks:${nextRunTime}:${task.task_id}`, JSON.stringify(task));
}
});
}
/**
* Run Instant Task
*
* @returns {void}
*/
processInstantTask() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
if (!this.taskQueue.observers.length)
return;
const [firstInstantKey] = yield this.redisService.zrange(task_trigger_types_enum_1.TaskTriggerTypes.INSTANT, 0, 1);
const [processingTaskString] = yield this.redisService.getMulti([`tasks:PROCESSING:${task_trigger_types_enum_1.TaskTriggerTypes.INSTANT}`]);
if (processingTaskString) {
const processingTask = JSON.parse(processingTaskString);
if (processingTask.task_instant_timeout != -1 &&
moment(processingTask.task_created_at).add(processingTask.task_instant_timeout || 30, 'seconds').isBefore(moment())) {
yield this.redisService.delete([`tasks:PROCESSING:${task_trigger_types_enum_1.TaskTriggerTypes.INSTANT}`]);
return this.processInstantTask();
}
}
if (!firstInstantKey || processingTaskString)
return;
const [taskString] = yield this.redisService.getMulti([firstInstantKey]);
if (!taskString) {
/** Remove zrange */
if (firstInstantKey) {
yield this.redisService.zrangeRemoveMember(task_trigger_types_enum_1.TaskTriggerTypes.INSTANT, firstInstantKey);
}
return;
}
const task = JSON.parse(taskString);
// Do nothing if the task assigned to another consumer
if (task.task_consumer_identity && this.consumerIdentity != task.task_consumer_identity)
return;
// Catch the task, prevent duplicates from other instances
// Only 1 instance can successfully delete the record
const getTaskSuccess = yield this.redisService.delete([firstInstantKey]);
if (!getTaskSuccess)
return; /** Other consumer caught the task */
if (this.InitialOptions.debug)
console.debug('Catch Task Success: ', task.task_name);
// Also delete key in index
yield this.redisService.deleteKeyInIndex(firstInstantKey, task_trigger_types_enum_1.TaskTriggerTypes.INSTANT);
yield this.redisService.set(processing_instant_task_key_const_1.PROCESSING_INSTANT_TASK_KEY, taskString);
(_a = this.taskQueue.observers[this.selectedObserver]) === null || _a === void 0 ? void 0 : _a.next(this.formatTaskCallback(task));
});
}
/**
* Log Consumer
*/
logConsumer() {
this.redisService.set(`consumer:${this.instanceId}:count`, JSON.stringify({
utc_offset: new Date().getTimezoneOffset() / -60,
platform: os.platform(),
total_mem: os.totalmem(),
free_mem: os.freemem(),
cpus: os.cpus().map(cpu => cpu.model),
count: this.taskQueue.observers.length,
identity: this.consumerIdentity,
instance_id: this.instanceId,
updated_at: moment().toISOString()
}));
}
/**
* Lookup Tasks
*
* @returns {void}
*/
lookup() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
this.logConsumer();
/** No Subscriber, retry later */
if (!this.taskQueue.observers.length)
return this.InitialOptions.debug && console.error("No subscribed consumer found.");
this.processInstantTask();
// Get Redis Keys
const timeKey = moment().format("YYYY:MM:DD:HH:mm:ss"), key = `tasks:${timeKey}:*`;
const keys = yield this.redisService.searchKeys(key);
if (!keys.length)
return;
// Get Tasks
const taskStrings = yield this.redisService.getMulti(keys);
if (!taskStrings.length)
return;
const tasks = taskStrings.map(taskString => JSON.parse(taskString));
// Log Consumer Status
if (this.InitialOptions.debug)
console.debug('Tasks to process: ', tasks.map(task => task.task_name), `Observers = ${this.taskQueue.observers.length}`);
// Loop Through Each Task
for (const task of tasks) {
// Catch the task, prevent duplicates from other instances
// Only 1 instance can successfully delete the record
const getTaskSuccess = yield this.redisService.delete([`tasks:${timeKey}:${task.task_id}`]);
if (!getTaskSuccess)
continue;
if (this.InitialOptions.debug)
console.debug('Catch Task Success: ', task.task_name);
// Pick an observer
if (this.selectedObserver + 1 >= this.taskQueue.observers.length)
this.selectedObserver = 0;
else
this.selectedObserver++;
(_a = this.taskQueue.observers[this.selectedObserver]) === null || _a === void 0 ? void 0 : _a.next(this.formatTaskCallback(task));
this.rotateTask(task);
}
});
}
/**
* Handle Task Failure
*
* @param task PDMQ Task
*/
handleTaskFailure(task) {
return __awaiter(this, void 0, void 0, function* () {
try {
const sourceTask = JSON.parse(JSON.stringify(task));
/** Create new task id */
sourceTask.task_id = uuid_1.v1();
/** verify task retry limit */
if (typeof sourceTask.task_fallback_retry_limit == 'number' && sourceTask.task_fallback_retry_limit > 0) {
// reduce limit
sourceTask.task_fallback_retry_limit -= 1;
/** add to time based queue */
if (moment(sourceTask.trigger_next_at).isValid()
&& typeof sourceTask.task_fallback_retry_duration == 'number'
&& sourceTask.task_fallback_retry_duration > 0) {
/** calculate next runtime */
sourceTask.task_trigger_type = task_trigger_types_enum_1.TaskTriggerTypes.ONCE;
sourceTask.task_trigger_once_datetime = moment().add(sourceTask.task_fallback_retry_duration, 's').toISOString();
yield this.addTask(sourceTask);
}
/** add to instant queue */
else {
sourceTask.task_trigger_type = task_trigger_types_enum_1.TaskTriggerTypes.INSTANT;
yield this.addTask(sourceTask);
}
}
/** run backup task if exist */
else if (task.task_fallback_task_id) {
/** Add stored task to instant queue */
yield this.taskService.runStoredTask(task.task_fallback_task_id, this.redisService);
}
}
catch (error) {
console.error(`[PDMQ] Unable handle task (${task.task_name} - ${task.task_id}) failure: `, error);
}
});
}
/**
* Add Task
*
* @param task
* @returns {PDMQTask}
*/
addTask(task) {
return this.taskService.addTask(task, this.redisService);
}
/**
*
* Format task callback
*
* @param task
* @returns
*/
formatTaskCallback(task) {
task.success = (res) => {
/** Remove processing key if instant task */
if (task.task_trigger_type === task_trigger_types_enum_1.TaskTriggerTypes.INSTANT)
this.redisService.delete([processing_instant_task_key_const_1.PROCESSING_INSTANT_TASK_KEY]);
/** Store result into redis and await for pdmq-ui to pickup */
if (task.task_exec_id) {
this.redisService.set(`tasks:COMPLETED:${task.task_exec_id}`, JSON.stringify({
success: true,
res,
task
}));
}
};
task.failed = (error) => {
/** Remove processing key if instant task */
if (task.task_trigger_type === task_trigger_types_enum_1.TaskTriggerTypes.INSTANT)
this.redisService.delete([processing_instant_task_key_const_1.PROCESSING_INSTANT_TASK_KEY]);
/** Store result into redis and await for pdmq-ui to pickup */
if (task.task_exec_id) {
this.redisService.set(`tasks:COMPLETED:${task.task_exec_id}`, JSON.stringify({
success: false,
error,
task
}));
}
/** Handle Failure */
this.handleTaskFailure(task);
};
return task;
}
}
exports.PDMQConsumer = PDMQConsumer;