@golemio/pid
Version:
Golemio PID Module
115 lines • 6.15 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProcessAndSendLogsTask = void 0;
const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
const IoRedisConnector_1 = require("@golemio/core/dist/helpers/data-access/redis/IoRedisConnector");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const ioc_1 = require("@golemio/core/dist/integration-engine/ioc");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const redis_semaphore_1 = require("@golemio/core/dist/shared/redis-semaphore");
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
const LogDateTimeUtils_1 = require("../helpers/LogDateTimeUtils");
const constants_1 = require("../constants");
const DEFAULT_BATCH_SIZE = 1000;
const DEFAULT_SEND_LOGS_LOCK_TIMEOUT = 60 * 1000;
let ProcessAndSendLogsTask = class ProcessAndSendLogsTask extends workers_1.AbstractEmptyTask {
constructor(logRepository, monitoringService, redisClient, logger, config) {
super(constants_1.PRESETS_WORKER_NAME);
this.logRepository = logRepository;
this.monitoringService = monitoringService;
this.redisClient = redisClient;
this.logger = logger;
this.config = config;
this.queueName = "processAndSendLogs";
this.queueTtl = 59 * 1000; // 59 seconds
this.LOCK_KEY_PHRASE = "ropidGTFSProcessAndSendLogsTask";
const batchSize = this.config.getValue("env.ROPID_PRESET_LOG_BATCH_SIZE", DEFAULT_BATCH_SIZE.toString());
this.batchSize = Number.parseInt(batchSize);
const lockTimeout = this.config.getValue("env.ROPID_PRESET_SEND_LOGS_LOCK_TIMEOUT", DEFAULT_SEND_LOGS_LOCK_TIMEOUT.toString());
this.lockTimeout = Number.parseInt(lockTimeout);
this.refreshInterval = this.lockTimeout * 0.8; // recommended
}
async execute() {
const mutex = this.createMutex();
const lockAcquired = await mutex.tryAcquire();
if (!lockAcquired) {
this.logger.error(`${this.constructor.name}: unable to obtain mutex lock.`);
return;
}
try {
const logEntries = await this.logRepository.findUnprocessed();
if (logEntries.length === 0) {
this.logger.warn(`${this.constructor.name}: no logs found to send to the ROPID monitoring system`);
return;
}
const batchCount = Math.ceil(logEntries.length / this.batchSize);
// Process logs in batches to avoid exceeding the maximum allowed size of the request
for (let i = 0; i < logEntries.length; i += this.batchSize) {
try {
const batchedLogs = logEntries.slice(i, i + this.batchSize);
await this.processBatch(batchedLogs, 1 + Math.ceil(i / this.batchSize), batchCount);
}
catch (err) {
this.logger.warn(err, `${this.constructor.name}: error while processing DCIP system logs`);
}
}
}
finally {
await mutex.release();
}
}
async processBatch(logEntries, batchIndex, batchCount) {
let logIds = [];
let logData = [];
for (const logEntry of logEntries) {
const { device_alias, received_at } = logEntry.toJSON();
logIds.push(logEntry.get("id"));
logData.push({
deviceAlias: device_alias,
receivedAt: LogDateTimeUtils_1.LogDateTimeUtils.parseISONoFractionalFromDate(received_at),
});
}
try {
await this.monitoringService.sendLogsToMonitoringSystem(logData);
}
catch (err) {
await this.logRepository.markAsBounced(logIds);
throw err;
}
this.logger.info({ batchIndex, batchCount }, `${this.constructor.name}: ${logData.length} logs sent to the ROPID monitoring system`);
await this.logRepository.markAsProcessed(logIds);
}
createMutex() {
return new redis_semaphore_1.Mutex(this.redisClient.getConnection(), this.LOCK_KEY_PHRASE, {
acquireAttemptsLimit: 1,
lockTimeout: this.lockTimeout,
refreshInterval: this.refreshInterval,
onLockLost: (err) => {
this.logger.error(err, `${this.constructor.name}: mutex lock was lost`);
},
});
}
};
exports.ProcessAndSendLogsTask = ProcessAndSendLogsTask;
exports.ProcessAndSendLogsTask = ProcessAndSendLogsTask = __decorate([
(0, tsyringe_1.injectable)(),
__param(0, (0, tsyringe_1.inject)(ModuleContainerToken_1.ModuleContainerToken.PresetLogRepository)),
__param(1, (0, tsyringe_1.inject)(ModuleContainerToken_1.ModuleContainerToken.RopidMonitoringService)),
__param(2, (0, tsyringe_1.inject)(ioc_1.ContainerToken.RedisConnector)),
__param(3, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
__param(4, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.SimpleConfig)),
__metadata("design:paramtypes", [Object, Object, IoRedisConnector_1.IoRedisConnector, Object, Object])
], ProcessAndSendLogsTask);
//# sourceMappingURL=ProcessAndSendLogsTask.js.map