tardis-dev
Version:
Convenient access to tick-level historical and real-time cryptocurrency market data via Node.js
286 lines • 14.7 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.replayNormalized = exports.replay = void 0;
const fs_extra_1 = require("fs-extra");
const path_1 = __importDefault(require("path"));
const stream_1 = require("stream");
const worker_threads_1 = require("worker_threads");
const zlib_1 = require("zlib");
const binarysplit_1 = require("./binarysplit");
const clearcache_1 = require("./clearcache");
const consts_1 = require("./consts");
const debug_1 = require("./debug");
const handy_1 = require("./handy");
const mappers_1 = require("./mappers");
const options_1 = require("./options");
async function* replay({ exchange, from, to, filters, skipDecoding = undefined, withDisconnects = undefined, apiKey = undefined, withMicroseconds = undefined, autoCleanup = undefined, waitWhenDataNotYetAvailable = undefined }) {
validateReplayOptions(exchange, from, to, filters);
const fromDate = (0, handy_1.parseAsUTCDate)(from);
const toDate = (0, handy_1.parseAsUTCDate)(to);
const cachedSlicePaths = new Map();
let replayError;
(0, debug_1.debug)('replay for exchange: %s started - from: %s, to: %s, filters: %o', exchange, fromDate.toISOString(), toDate.toISOString(), filters);
const options = (0, options_1.getOptions)();
// initialize worker thread that will fetch and cache data feed slices and "report back" by setting proper key/values in cachedSlicePaths
const payload = {
cacheDir: options.cacheDir,
endpoint: options.endpoint,
apiKey: apiKey || options.apiKey,
userAgent: options._userAgent,
fromDate,
toDate,
exchange,
filters: filters || [],
waitWhenDataNotYetAvailable
};
const worker = new ReliableWorker(payload);
worker.on('message', (message) => {
cachedSlicePaths.set(message.sliceKey, message.slicePath);
});
worker.on('error', (err) => {
(0, debug_1.debug)('worker error %o', err);
replayError = err;
});
try {
// date is always formatted to have length of 28 so we can skip looking for first space in line and use it
// as hardcoded value
const DATE_MESSAGE_SPLIT_INDEX = 28;
// more lenient gzip decompression
// see https://github.com/request/request/pull/2492 and https://github.com/node-fetch/node-fetch/pull/239
const ZLIB_OPTIONS = {
chunkSize: 128 * 1024,
flush: zlib_1.constants.Z_SYNC_FLUSH,
finishFlush: zlib_1.constants.Z_SYNC_FLUSH
};
// helper flag that helps us not yielding two subsequent undefined/disconnect messages
let lastMessageWasUndefined = false;
let currentSliceDate = new Date(fromDate);
// iterate over every minute in <=from,to> date range
// get cached slice paths, read them as file streams, decompress, split by new lines and yield as messages
while (currentSliceDate < toDate) {
const sliceKey = currentSliceDate.toISOString();
(0, debug_1.debug)('getting slice: %s, exchange: %s', sliceKey, exchange);
let cachedSlicePath;
while (cachedSlicePath === undefined) {
cachedSlicePath = cachedSlicePaths.get(sliceKey);
// if something went wrong(network issue, auth issue, gunzip issue etc)
if (replayError !== undefined) {
throw replayError;
}
if (cachedSlicePath === undefined) {
// if response for requested date is not ready yet wait 100ms and try again
(0, debug_1.debug)('waiting for slice: %s, exchange: %s', sliceKey, exchange);
await (0, handy_1.wait)(100);
}
}
// response is a path to file on disk let' read it as stream
const linesStream = (0, fs_extra_1.createReadStream)(cachedSlicePath, { highWaterMark: 128 * 1024 })
// unzip it
.pipe((0, zlib_1.createGunzip)(ZLIB_OPTIONS))
.on('error', function onGunzipError(err) {
(0, debug_1.debug)('gunzip error %o', err);
linesStream.destroy(err);
})
// and split by new line
.pipe(new binarysplit_1.BinarySplitStream())
.on('error', function onBinarySplitStreamError(err) {
(0, debug_1.debug)('binary split stream error %o', err);
linesStream.destroy(err);
});
let linesCount = 0;
for await (const bufferLine of linesStream) {
linesCount++;
if (bufferLine.length > 0) {
lastMessageWasUndefined = false;
const localTimestampBuffer = bufferLine.slice(0, DATE_MESSAGE_SPLIT_INDEX);
const messageBuffer = bufferLine.slice(DATE_MESSAGE_SPLIT_INDEX + 1);
// as any due to https://github.com/Microsoft/TypeScript/issues/24929
if (skipDecoding === true) {
yield {
localTimestamp: localTimestampBuffer,
message: messageBuffer
};
}
else {
let messageString = messageBuffer.toString();
// hack to handle huobi long numeric id for trades
if (exchange.startsWith('huobi-') && messageString.includes('.trade.detail')) {
messageString = messageString.replace(/"id":([0-9]+),/g, '"id":"$1",');
}
// hack to handle upbit long numeric id for trades
if (exchange === 'upbit' && messageString.includes('sequential_id')) {
messageString = messageString.replace(/"sequential_id":([0-9]+),/g, '"sequential_id":"$1",');
}
const message = JSON.parse(messageString);
const localTimestampString = localTimestampBuffer.toString();
const localTimestamp = new Date(localTimestampString);
if (withMicroseconds) {
// provide additionally fractions of millisecond at microsecond resolution
// local timestamp always has format like this 2019-06-01T00:03:03.1238784Z
localTimestamp.μs = (0, handy_1.parseμs)(localTimestampString);
}
yield {
// when skipDecoding is not set, decode timestamp to Date and message to object
localTimestamp,
message
};
}
// ignore empty lines unless withDisconnects is set to true
// do not yield subsequent undefined messages
}
else if (withDisconnects === true && lastMessageWasUndefined === false) {
lastMessageWasUndefined = true;
yield undefined;
}
}
(0, debug_1.debug)('processed slice: %s, exchange: %s, count: %d', sliceKey, exchange, linesCount);
// remove slice key from the map as it's already processed
cachedSlicePaths.delete(sliceKey);
if (autoCleanup) {
await cleanupSlice(cachedSlicePath);
}
// move one minute forward
currentSliceDate.setUTCMinutes(currentSliceDate.getUTCMinutes() + 1);
}
(0, debug_1.debug)('replay for exchange: %s finished - from: %s, to: %s, filters: %o', exchange, fromDate.toISOString(), toDate.toISOString(), filters);
}
finally {
if (autoCleanup) {
(0, debug_1.debug)('replay for exchange %s auto cleanup started - from: %s, to: %s, filters: %o', exchange, fromDate.toISOString(), toDate.toISOString(), filters);
let startDate = new Date(fromDate);
while (startDate < toDate) {
(0, clearcache_1.clearCacheSync)(exchange, filters, startDate.getUTCFullYear(), startDate.getUTCMonth() + 1, startDate.getUTCDate());
startDate = (0, handy_1.addDays)(startDate, 1);
}
(0, debug_1.debug)('replay for exchange %s auto cleanup finished - from: %s, to: %s, filters: %o', exchange, fromDate.toISOString(), toDate.toISOString(), filters);
}
const underlyingWorker = worker.getUnderlyingWorker();
if (underlyingWorker !== undefined) {
await terminateWorker(underlyingWorker, 500);
}
}
}
exports.replay = replay;
async function cleanupSlice(slicePath) {
try {
await (0, fs_extra_1.remove)(slicePath);
}
catch (e) {
(0, debug_1.debug)('cleanupSlice error %s %o', slicePath, e);
}
}
// gracefully terminate worker
async function terminateWorker(worker, waitTimeout) {
let cancelWait = () => { };
const maxWaitGuard = new Promise((resolve) => {
const timeoutId = setTimeout(resolve, waitTimeout);
cancelWait = () => clearTimeout(timeoutId);
});
const readyToTerminate = new Promise((resolve) => {
worker.once('message', (signal) => signal === "READY_TO_TERMINATE" /* WorkerSignal.READY_TO_TERMINATE */ && resolve());
}).then(cancelWait);
worker.postMessage("BEFORE_TERMINATE" /* WorkerSignal.BEFORE_TERMINATE */);
await Promise.race([readyToTerminate, maxWaitGuard]);
await worker.terminate();
}
function replayNormalized({ exchange, symbols, from, to, withDisconnectMessages = undefined, apiKey = undefined, autoCleanup = undefined, waitWhenDataNotYetAvailable = undefined }, ...normalizers) {
const fromDate = (0, handy_1.parseAsUTCDate)(from);
validateReplayNormalizedOptions(fromDate, normalizers);
//TODO: zrovi replay dzien po dniu, tak ze kazdego dnia przekazuje swierze filters
const createMappers = (localTimestamp) => normalizers.map((m) => m(exchange, localTimestamp));
const mappers = createMappers(fromDate);
const filters = (0, handy_1.getFilters)(mappers, symbols);
const messages = replay({
exchange,
from,
to,
withDisconnects: true,
filters,
apiKey,
withMicroseconds: true,
autoCleanup,
waitWhenDataNotYetAvailable
});
// filter normalized messages by symbol as some exchanges do not provide server side filtering so we could end up with messages
// for symbols we've not requested for
const upperCaseSymbols = symbols !== undefined ? symbols.map((s) => s.toUpperCase()) : undefined;
const filter = (symbol) => {
return upperCaseSymbols === undefined || upperCaseSymbols.length === 0 || upperCaseSymbols.includes(symbol);
};
return (0, handy_1.normalizeMessages)(exchange, undefined, messages, mappers, createMappers, withDisconnectMessages, filter);
}
exports.replayNormalized = replayNormalized;
function validateReplayOptions(exchange, from, to, filters) {
if (!exchange || consts_1.EXCHANGES.includes(exchange) === false) {
throw new Error(`Invalid "exchange" argument: ${exchange}. Please provide one of the following exchanges: ${consts_1.EXCHANGES.join(', ')}.`);
}
if (!from || isNaN(Date.parse(from))) {
throw new Error(`Invalid "from" argument: ${from}. Please provide valid date string.`);
}
if (!to || isNaN(Date.parse(to))) {
throw new Error(`Invalid "to" argument: ${to}. Please provide valid date string.`);
}
if ((0, handy_1.parseAsUTCDate)(to) < (0, handy_1.parseAsUTCDate)(from)) {
throw new Error(`Invalid "to" and "from" arguments combination. Please provide "to" date that is later than "from" date.`);
}
if (filters && filters.length > 0) {
for (let i = 0; i < filters.length; i++) {
const filter = filters[i];
if (!filter.channel || consts_1.EXCHANGE_CHANNELS_INFO[exchange].includes(filter.channel) === false) {
throw new Error(`Invalid "filters[].channel" argument: ${filter.channel}. Please provide one of the following channels: ${consts_1.EXCHANGE_CHANNELS_INFO[exchange].join(', ')}.`);
}
if (filter.symbols && Array.isArray(filter.symbols) === false) {
throw new Error(`Invalid "filters[].symbols" argument: ${filter.symbols}. Please provide array of symbol strings`);
}
}
}
}
function validateReplayNormalizedOptions(fromDate, normalizers) {
const hasBookChangeNormalizer = normalizers.some((n) => n === mappers_1.normalizeBookChanges);
const dateDoesNotStartAtTheBeginningOfTheDay = fromDate.getUTCHours() !== 0 || fromDate.getUTCMinutes() !== 0;
if (hasBookChangeNormalizer && dateDoesNotStartAtTheBeginningOfTheDay) {
(0, debug_1.debug)('Initial order book snapshots are available only at 00:00 UTC');
}
}
class ReliableWorker extends stream_1.EventEmitter {
constructor(_payload) {
super();
this._payload = _payload;
this._errorsCount = 0;
this._worker = undefined;
this._handleError = async (err) => {
(0, debug_1.debug)('underlying worker error %o', err);
if (err.message.includes('HttpError') === false && this._errorsCount < 30) {
this._errorsCount++;
const delayMS = Math.min(Math.pow(2, this._errorsCount) * 1000, 120 * 1000);
(0, debug_1.debug)('re-init worker after: %d ms', delayMS);
await (0, handy_1.wait)(delayMS);
// it was most likely unhandled socket hang up error, let's retry first with new worker and don't emit error right away
this._initWorker();
}
else {
this.emit('error', err);
}
};
this._initWorker();
}
_initWorker() {
this._worker = new worker_threads_1.Worker(path_1.default.resolve(__dirname, 'worker.js'), {
workerData: this._payload
});
this._worker.on('message', (message) => {
this.emit('message', message);
});
this._worker.on('error', this._handleError);
this._worker.on('exit', (code) => {
(0, debug_1.debug)('worker finished with code: %d', code);
});
}
getUnderlyingWorker() {
return this._worker;
}
}
//# sourceMappingURL=replay.js.map