@bazilio-san/af-stream
Version:
Data stream from database table
421 lines (419 loc) • 19.1 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Stream = void 0;
const luxon_1 = require("luxon");
const cron = require("cron");
const mssql_1 = require("mssql");
const LastTimeRecords_1 = require("./LastTimeRecords");
const RecordsBuffer_1 = require("./RecordsBuffer");
const StartTimeRedis_1 = require("./StartTimeRedis");
const VirtualTimeObj_1 = require("./VirtualTimeObj");
const utils_1 = require("./utils/utils");
const db_1 = require("./db/db");
const color_1 = require("./utils/color");
const get_sender_1 = require("./sender/get-sender");
const constants_1 = require("./constants");
class Stream {
constructor(options) {
var _a;
this.sessionId = `sid${+(new Date())}`;
this.locked = false;
this.initialized = false;
this.isFirstLoad = true;
const { streamConfig, prepareEvent, tsFieldToMillis, millis2dbFn, loopTime = 0 } = options;
const { fetchIntervalSec, bufferMultiplier, src, maxBufferSize } = streamConfig;
src.timezoneOfTsField = src.timezoneOfTsField || 'GMT';
const zone = src.timezoneOfTsField;
this.options = options;
const tsFieldToMillisDefault = (tsValue) => {
if (typeof tsValue === 'string') {
return luxon_1.DateTime.fromISO(tsValue, { zone }).toMillis();
}
return Number(tsValue);
};
this.tsFieldToMillis = typeof tsFieldToMillis === 'function'
? tsFieldToMillis.bind(this)
: tsFieldToMillisDefault;
this.prepareEvent = typeof prepareEvent === 'function'
? prepareEvent.bind(this)
: (dbRecord) => dbRecord;
this.millis2dbFn = typeof millis2dbFn === 'function'
? millis2dbFn.bind(this)
: (millis) => `'${(0, utils_1.millis2iso)(millis)}'`;
const { idFields } = src;
this.bufferLookAheadMs = ((fetchIntervalSec || 10) * 1000 * (bufferMultiplier || 30));
this.maxBufferSize = maxBufferSize || 65536;
this.sender = {};
this.db = {};
this.lastRecordTs = 0;
this.loopTimeMillis = (0, utils_1.getTimeParamMillis)(loopTime);
this.recordsBuffer = new RecordsBuffer_1.RecordsBuffer();
/*
A set of hashes of string identification fields, along with a timestamp
equal to the largest value in the last received packet.
Serves to discard from the next portion of the data that has already been loaded.
This is necessary if there can be several entries for one timestamp.
EXAMPLE:
tradeno tradetime orderno seccode buysell client
38686190 2022-02-07 10:29:55.0000000 3420385 FSTOSS300901C00000010 B MCU1100
38686190 2022-02-07 10:29:55.0000000 3420375 FSTOSS300901C00000010 S MCU57801
In order to guarantee not to lose data, we request them with a timestamp overlap
WHERE [${tsField}] >= '${from}' AND [${tsField}] <= '${to}'
To ensure that duplicates are excluded, after receiving the data, we delete from there those that are in lastTimeRecords
*/
this.lastTimeRecords = new LastTimeRecords_1.LastTimeRecords(idFields);
this.virtualTimeObj = {};
this.sendTimer = null;
this.sendInterval = 10; // ms
this.totalRowsSent = 0;
this.busy = 0;
(_a = options.eventEmitter) === null || _a === void 0 ? void 0 : _a.on('virtual-time-loop-back', () => {
this.lastRecordTs = 0;
this.recordsBuffer.flush();
this.lastTimeRecords.flush();
this.totalRowsSent = 0;
this.isFirstLoad = true;
});
this.prefix = `${color_1.lCyan}STREAM: ${color_1.lBlue}${options.streamConfig.streamId}${color_1.rs}`;
}
async init() {
const { options, loopTimeMillis, millis2dbFn } = this;
const { senderConfig, eventEmitter, echo, logger, redis, serviceName, streamConfig, useStartTimeFromRedisCache, exitOnError, testMode, } = options;
let { speed } = options;
if (/^[\d.]+$/.test(String(speed))) {
speed = Math.min(Math.max(0.2, parseFloat(String(speed))), 5000);
}
else {
speed = 1;
}
const senderConstructorOptions = {
streamConfig,
senderConfig,
serviceName,
echo,
logger,
exitOnError,
eventEmitter,
};
this.sender = await (0, get_sender_1.default)(senderConstructorOptions);
const isConnectedToTarget = await this.sender.connect();
if (!isConnectedToTarget) {
exitOnError('No connection to sender');
return;
}
const { host, port } = redis;
const { src: { dbOptions, dbConfig, timezoneOfTsField }, streamId, fetchIntervalSec } = streamConfig;
const startTimeRedisOptions = {
useStartTimeFromRedisCache,
host,
port,
streamId,
eventEmitter,
exitOnError,
logger,
};
const startTimeRedis = new StartTimeRedis_1.StartTimeRedis(startTimeRedisOptions);
const { isUsedSavedStartTime, startTime } = await startTimeRedis.getStartTime();
const virtualTimeObjOptions = {
startTime,
speed,
loopTimeMillis,
eventEmitter,
exitOnError,
};
this.virtualTimeObj = (0, VirtualTimeObj_1.getVirtualTimeObj)(virtualTimeObjOptions);
const eqFill = '='.repeat(Math.max(1, (36 - streamId.length) / 2));
const info = `${color_1.g}${eqFill} [@bazilio-san/af-stream: ${streamId}] ${eqFill}
${color_1.g}Time field TZ: ${color_1.m}${timezoneOfTsField}
${color_1.g}Start from beginning: ${color_1.m}${useStartTimeFromRedisCache ? 'NOT' : 'YES'}
${color_1.g}Start time: ${color_1.m}${(0, utils_1.millis2iso)(startTime)}${isUsedSavedStartTime ? `${color_1.y}${color_1.bold} TAKEN FROM CACHE${color_1.boldOff}${color_1.rs}${color_1.g}` : ''}
${color_1.g}Speed: ${color_1.m}${this.virtualTimeObj.speed}x
${color_1.g}Cyclicity: ${color_1.m}${loopTimeMillis ? `${loopTimeMillis / 1000} sec` : '-'}
${color_1.g}Db polling frequency: ${color_1.m}${fetchIntervalSec} sec
${color_1.g}================================================================`;
echo(info);
if (!testMode) {
const dbConstructorOptions = {
streamConfig,
logger,
eventEmitter,
exitOnError,
dbOptions,
dbConfig,
millis2dbFn,
};
this.db = await (0, db_1.default)(dbConstructorOptions);
}
this.initialized = true;
return this;
}
async start() {
if (!this.initialized) {
await this.init();
}
await this._loadNextPortion();
this._fetchLoop();
this._printInfoLoop();
// Additional external call loop in case of interruption of the chain of internal calls _sendLoop()
setInterval(() => {
this._sendLoop().then(() => null);
}, 1000);
return this;
}
// Greatest index of a value less than the specified
findEndIndex() {
const virtualTime = this.virtualTimeObj.getVirtualTs();
/*
if (DEBUG_STREAM) {
const { buffer: rb } = this.recordsBuffer;
const firstISO = rb.length ? millis2iso(rb[0][TS_FIELD]) : '-';
const lastISO = rb.length > 1 ? millis2iso(rb[rb.length - 1][TS_FIELD]) : '-';
this.options.echo(`findEndIndex() ${c}virtualTime: ${m}${millis2iso(virtualTime)}${rs} [${m}${firstISO}${rs} - ${m}${lastISO}${rs}]`);
}
*/
return this.recordsBuffer.findIndexOfNearestSmaller(virtualTime);
}
static packetInfo(count, fromRecord, toRecord) {
if (count && fromRecord && toRecord) {
const HMS = 'HH:mm:ss.SSS';
const from = fromRecord[constants_1.TS_FIELD];
const to = toRecord[constants_1.TS_FIELD];
const fromLu = luxon_1.DateTime.fromMillis(from);
const timeRange = `${fromLu.toFormat('LL-dd')} ${fromLu.toFormat(HMS)} - ${luxon_1.DateTime.fromMillis(to).toFormat(HMS)}`;
return `r: ${(0, utils_1.padL)(count, 5)} / ${timeRange} / ${(0, utils_1.padL)(`${to - from} ms`, 10)}`;
}
return ' '.repeat(60);
}
async prepareEventsPacket(dbRecordOrRecordset) {
const { options: { streamConfig: { src: { tsField } } }, prepareEvent, tsFieldToMillis } = this;
if (!Array.isArray(dbRecordOrRecordset)) {
if (!dbRecordOrRecordset || typeof dbRecordOrRecordset !== 'object') {
return [];
}
dbRecordOrRecordset = [dbRecordOrRecordset];
}
return mssql_1.Promise.all(dbRecordOrRecordset.map((record) => {
record[constants_1.TS_FIELD] = tsFieldToMillis(record[tsField]);
return prepareEvent(record);
}));
}
async _addPortionToBuffer(recordset) {
var _a, _b;
const { recordsBuffer, loopTimeMillis, options } = this;
const { streamConfig: { streamId } } = options;
const { length: loaded = 0 } = recordset;
let skipped = 0;
let toUse = loaded;
if (loaded) {
const forBuffer = await this.prepareEventsPacket(recordset);
if (loopTimeMillis) {
const bias = Date.now() - this.virtualTimeObj.realStartTsLoopSafe;
forBuffer.forEach((row) => {
row._ts = row[constants_1.TS_FIELD] + bias;
row.loopNumber = this.virtualTimeObj.loopNumber;
});
}
const lastRecordTsBeforeCheck = forBuffer[forBuffer.length - 1][constants_1.TS_FIELD];
const subtractedLastTimeRecords = this.lastTimeRecords.subtractLastTimeRecords(forBuffer);
if (constants_1.DEBUG_LTR) {
const payload = { streamId, subtractedLastTimeRecords };
(_a = options.eventEmitter) === null || _a === void 0 ? void 0 : _a.emit('subtracted-last-time-records', payload);
}
toUse = forBuffer.length;
if (toUse !== loaded) {
skipped = loaded - toUse;
}
if (toUse) {
recordsBuffer.add(forBuffer);
this.lastRecordTs = recordsBuffer.lastTs;
const currentLastTimeRecords = this.lastTimeRecords.fillLastTimeRecords(this.recordsBuffer.buffer);
if (constants_1.DEBUG_LTR) {
const payload = { streamId, currentLastTimeRecords };
(_b = options.eventEmitter) === null || _b === void 0 ? void 0 : _b.emit('current-last-time-records', payload);
}
}
else {
this.lastRecordTs = lastRecordTsBeforeCheck + 1;
}
}
if (constants_1.DEBUG_STREAM) {
options.echo(`${this.prefix} vt: ${this.virtualTimeObj.getString()} loaded/skipped/used: ${color_1.lm}${loaded}${color_1.blue}/${color_1.lc}${skipped}${color_1.blue}/${color_1.g}${toUse}${color_1.rs}`);
}
}
async _loadNextPortion() {
var _a, _b;
const { options, recordsBuffer, virtualTimeObj: vtObj, bufferLookAheadMs, lastRecordTs, maxBufferSize } = this;
const { streamConfig: { streamId } } = options;
const virtualTimeObj = vtObj;
let startTs;
let endTs;
if (this.isFirstLoad) {
startTs = virtualTimeObj.virtualStartTs;
endTs = startTs + bufferLookAheadMs;
this.isFirstLoad = false;
}
else {
startTs = lastRecordTs || virtualTimeObj.virtualStartTs;
endTs = virtualTimeObj.getVirtualTs() + bufferLookAheadMs;
}
if (startTs >= endTs) {
return;
}
// Если расстояние по времени от первой до последней записи в буфере больше bufferLookAheadMs, новых записей подгружать не нужно
if (((recordsBuffer.getMsDistance()) > bufferLookAheadMs)) {
return;
}
const limit = maxBufferSize - recordsBuffer.buffer.length;
if (limit < 1) {
return;
}
if (constants_1.DEBUG_LNP) {
options.echo(`${this.prefix} ${color_1.c}_loadNextPortion()${color_1.rs} vt: ${color_1.m}${virtualTimeObj.getString()}${color_1.rs} from: ${color_1.m}${(0, utils_1.millis2iso)(startTs)}${color_1.rs} to ${color_1.m}${(0, utils_1.millis2iso)(endTs)}${color_1.rs}`);
}
try {
if (constants_1.DEBUG_LNP) {
const payload = { streamId, startTs, endTs };
(_a = options.eventEmitter) === null || _a === void 0 ? void 0 : _a.emit('before-load-next-portion', payload);
}
const recordset = await this.db.getPortionOfData({ startTs, endTs, limit });
if (recordset.length) {
endTs = this.tsFieldToMillis(recordset[recordset.length - 1][options.streamConfig.src.tsField]);
}
await this._addPortionToBuffer(recordset);
if (constants_1.DEBUG_LNP) {
const payload = {
streamId,
startTs,
endTs,
lastRecordTs: this.lastRecordTs,
last: recordsBuffer.last,
vt: virtualTimeObj.getVirtualTs(),
};
(_b = options.eventEmitter) === null || _b === void 0 ? void 0 : _b.emit('after-load-next-portion', payload);
}
}
catch (err) {
err.message += `\n${this.db.schemaAndTable}`;
options.exitOnError(err);
}
}
_fetchLoop() {
const { options: { streamConfig } } = this;
cron.job(`0/${streamConfig.fetchIntervalSec || 10} * * * * *`, async () => {
if (this.locked) {
return;
}
if (this.busy === 0 || this.busy > 5) {
this.busy = 1;
try {
await this._loadNextPortion();
}
catch (err) {
this.options.exitOnError(err);
return;
}
this.busy = 0;
}
else {
this.busy++;
}
}, null, true, 'GMT', undefined, false);
// onComplete, start, timeZone, context, runOnInit
}
_printInfoLoop() {
const { streamConfig, logger } = this.options;
cron.job(`0/${streamConfig.printInfoIntervalSec || 30} * * * * *`, () => {
const rowsSent = `rows sent: ${color_1.bold}${(0, utils_1.padL)(this.totalRowsSent || 0, 6)}${color_1.boldOff}${color_1.rs}`;
logger.info(`${this.prefix} ${rowsSent} / ${this.virtualTimeObj.getString()}`);
}, null, true, 'GMT', undefined, false);
// onComplete, start, timeZone, context, runOnInit
}
async _sendPacket(eventsPacket) {
const { sender, sessionId, options: { eventEmitter, logger, streamConfig: { streamId } } } = this;
return new mssql_1.Promise((resolve) => {
let debugMessage = '';
setTimeout(() => {
if (constants_1.DEBUG_STREAM) {
debugMessage += `${this.prefix}`;
}
const first = eventsPacket[0];
const recordsComposite = {
sessionId,
streamId,
eventsPacket,
isSingleRecordAsObject: true,
first,
last: first,
};
sender.sendEvents(recordsComposite).then(() => {
const { last, sendCount = 0, sentBufferLength } = recordsComposite;
const lastTs = last === null || last === void 0 ? void 0 : last[constants_1.TS_FIELD];
if (lastTs) {
const payload = { streamId, lastTs };
eventEmitter.emit('save-last-ts', payload);
}
this.totalRowsSent += sendCount;
if (constants_1.DEBUG_STREAM) {
debugMessage += ` SENT: ${color_1.c}${Stream.packetInfo(sendCount, first, last)}`;
debugMessage += ` / ${(0, utils_1.padL)(sentBufferLength, 6)}b`;
debugMessage += ` / r.tot: ${color_1.bold}${(0, utils_1.padL)(this.totalRowsSent, 6)}${color_1.boldOff}${color_1.rs}`;
}
resolve({ debugMessage });
}).catch((err) => {
logger.error(err);
resolve({ debugMessage, isError: true });
});
}, 5);
});
}
async _send() {
const { recordsBuffer: rb, virtualTimeObj } = this;
if (!virtualTimeObj.ready) {
return;
}
const index = this.findEndIndex();
if (index < 0) {
return;
}
const eventsPacket = rb.shiftBy(index + 1);
let debugMessage;
if (eventsPacket.length) {
({ debugMessage } = await this._sendPacket(eventsPacket));
if (eventsPacket.length) {
rb.unshiftEvents(eventsPacket);
}
else {
rb.setEdges();
}
}
if (constants_1.DEBUG_STREAM) {
let bufferInfo = Stream.packetInfo(rb.length, rb.first, rb.last);
bufferInfo = bufferInfo.trim() ? `BUFFER: ${bufferInfo}` : `BUFFER empty`;
this.options.echo(`${debugMessage}\t${color_1.m}${bufferInfo}`);
}
}
async _sendLoop() {
const self = this;
clearTimeout(this.sendTimer);
try {
await this._send();
}
catch (err) {
return self.options.exitOnError(err);
}
this.sendTimer = setTimeout(() => {
self._sendLoop();
}, this.sendInterval);
}
setEventCallback(eventCallback) {
this.sender.eventCallback = eventCallback;
}
lock() {
this.locked = true;
}
unLock() {
this.locked = false;
}
}
exports.Stream = Stream;
//# sourceMappingURL=Stream.js.map