@jalmonter/tardis-dev
Version:
Convenient access to tick-level historical and real-time cryptocurrency market data via Node.js
301 lines • 10.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.fromMicroSecondsToDate = exports.lowerCaseSymbols = exports.upperCaseSymbols = exports.asNumberIfValid = exports.decimalPlaces = exports.CappedSet = exports.CircularBuffer = exports.cleanTempFiles = exports.optimizeFilters = exports.parseμs = exports.batch = exports.getFilters = exports.normalizeMessages = exports.take = exports.HttpError = exports.ONE_SEC_IN_MS = exports.sequence = exports.addDays = exports.addMinutes = exports.doubleDigit = exports.formatDateToPath = exports.wait = exports.parseAsUTCDate = void 0;
function parseAsUTCDate(val) {
// not sure about this one, but it should force parsing date as UTC date not as local timezone
if (val.endsWith('Z') === false) {
val += 'Z';
}
var date = new Date(val);
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours(), date.getUTCMinutes()));
}
exports.parseAsUTCDate = parseAsUTCDate;
function wait(delayMS) {
return new Promise((resolve) => {
setTimeout(resolve, delayMS);
});
}
exports.wait = wait;
function formatDateToPath(date) {
const year = date.getUTCFullYear();
const month = doubleDigit(date.getUTCMonth() + 1);
const day = doubleDigit(date.getUTCDate());
const hour = doubleDigit(date.getUTCHours());
const minute = doubleDigit(date.getUTCMinutes());
return `${year}/${month}/${day}/${hour}/${minute}`;
}
exports.formatDateToPath = formatDateToPath;
function doubleDigit(input) {
return input < 10 ? '0' + input : '' + input;
}
exports.doubleDigit = doubleDigit;
function addMinutes(date, minutes) {
return new Date(date.getTime() + minutes * 60000);
}
exports.addMinutes = addMinutes;
function addDays(date, days) {
return new Date(date.getTime() + days * 60000 * 1440);
}
exports.addDays = addDays;
function* sequence(end, seed = 0) {
let current = seed;
while (current < end) {
yield current;
current += 1;
}
return;
}
exports.sequence = sequence;
exports.ONE_SEC_IN_MS = 1000;
class HttpError extends Error {
constructor(status, responseText, url) {
super(`HttpError: status code: ${status}, response text: ${responseText}`);
this.status = status;
this.responseText = responseText;
this.url = url;
}
}
exports.HttpError = HttpError;
function* take(iterable, length) {
if (length === 0) {
return;
}
for (const item of iterable) {
yield item;
length--;
if (length === 0) {
return;
}
}
}
exports.take = take;
async function* normalizeMessages(exchange, messages, mappers, createMappers, withDisconnectMessages, filter, currentTimestamp) {
let previousLocalTimestamp = currentTimestamp;
let mappersForExchange = mappers;
if (mappersForExchange.length === 0) {
throw new Error(`Can't normalize data without any normalizers provided`);
}
for await (const messageWithTimestamp of messages) {
if (messageWithTimestamp === undefined) {
// we received undefined meaning Websocket disconnection
// lets create new mappers with clean state for 'new connection'
mappersForExchange = undefined;
// if flag withDisconnectMessages is set, yield disconnect message
if (withDisconnectMessages === true && previousLocalTimestamp !== undefined) {
const disconnect = {
type: 'disconnect',
exchange,
localTimestamp: previousLocalTimestamp
};
yield disconnect;
}
continue;
}
if (mappersForExchange === undefined) {
mappersForExchange = createMappers(messageWithTimestamp.localTimestamp);
}
previousLocalTimestamp = messageWithTimestamp.localTimestamp;
for (const mapper of mappersForExchange) {
if (mapper.canHandle(messageWithTimestamp.message)) {
const mappedMessages = mapper.map(messageWithTimestamp.message, messageWithTimestamp.localTimestamp);
if (!mappedMessages) {
continue;
}
for (const message of mappedMessages) {
if (filter === undefined) {
yield message;
}
else if (filter(message.symbol)) {
yield message;
}
}
}
}
}
}
exports.normalizeMessages = normalizeMessages;
function getFilters(mappers, symbols) {
const filters = mappers.flatMap((mapper) => mapper.getFilters(symbols));
const deduplicatedFilters = filters.reduce((prev, current) => {
const matchingExisting = prev.find((c) => c.channel === current.channel);
if (matchingExisting !== undefined) {
if (matchingExisting.symbols !== undefined && current.symbols) {
for (let symbol of current.symbols) {
if (matchingExisting.symbols.includes(symbol) === false) {
matchingExisting.symbols.push(symbol);
}
}
}
else if (current.symbols) {
matchingExisting.symbols = [...current.symbols];
}
}
else {
prev.push(current);
}
return prev;
}, []);
return deduplicatedFilters;
}
exports.getFilters = getFilters;
function* batch(symbols, batchSize) {
for (let i = 0; i < symbols.length; i += batchSize) {
yield symbols.slice(i, i + batchSize);
}
}
exports.batch = batch;
function parseμs(dateString) {
// check if we have ISO 8601 format date string, e.g: 2019-06-01T00:03:03.1238784Z or 2020-07-22T00:09:16.836773Z
// or 2020-03-01T00:00:24.893456+00:00
if (dateString.length === 27 || dateString.length === 28 || dateString.length === 32 || dateString.length === 30) {
return Number(dateString.slice(23, 26));
}
return 0;
}
exports.parseμs = parseμs;
function optimizeFilters(filters) {
// deduplicate filters (if the channel was provided multiple times)
const optimizedFilters = filters.reduce((prev, current) => {
const matchingExisting = prev.find((c) => c.channel === current.channel);
if (matchingExisting) {
// both previous and current have symbols let's merge them
if (matchingExisting.symbols && current.symbols) {
matchingExisting.symbols.push(...current.symbols);
}
else if (current.symbols) {
matchingExisting.symbols = [...current.symbols];
}
}
else {
prev.push(current);
}
return prev;
}, []);
// sort filters in place to improve local disk cache ratio (no matter filters order if the same filters are provided will hit the cache)
optimizedFilters.sort((f1, f2) => {
if (f1.channel < f2.channel) {
return -1;
}
if (f1.channel > f2.channel) {
return 1;
}
return 0;
});
// sort and deduplicate filters symbols
optimizedFilters.forEach((filter) => {
if (filter.symbols) {
filter.symbols = [...new Set(filter.symbols)].sort();
}
});
return optimizedFilters;
}
exports.optimizeFilters = optimizeFilters;
const tmpFileCleanups = new Map();
function cleanTempFiles() {
tmpFileCleanups.forEach((cleanup) => cleanup());
}
exports.cleanTempFiles = cleanTempFiles;
class CircularBuffer {
constructor(_bufferSize) {
this._bufferSize = _bufferSize;
this._buffer = [];
this._index = 0;
}
append(value) {
const isFull = this._buffer.length === this._bufferSize;
let poppedValue;
if (isFull) {
poppedValue = this._buffer[this._index];
}
this._buffer[this._index] = value;
this._index = (this._index + 1) % this._bufferSize;
return poppedValue;
}
*items() {
for (let i = 0; i < this._buffer.length; i++) {
const index = (this._index + i) % this._buffer.length;
yield this._buffer[index];
}
}
get count() {
return this._buffer.length;
}
clear() {
this._buffer = [];
this._index = 0;
}
}
exports.CircularBuffer = CircularBuffer;
class CappedSet {
constructor(_maxSize) {
this._maxSize = _maxSize;
this._set = new Set();
}
has(value) {
return this._set.has(value);
}
add(value) {
if (this._set.size >= this._maxSize) {
this._set.delete(this._set.keys().next().value);
}
this._set.add(value);
}
remove(value) {
this._set.delete(value);
}
size() {
return this._set.size;
}
}
exports.CappedSet = CappedSet;
function hasFraction(n) {
return Math.abs(Math.round(n) - n) > 1e-10;
}
// https://stackoverflow.com/a/44815797
function decimalPlaces(n) {
let count = 0;
// multiply by increasing powers of 10 until the fractional part is ~ 0
while (hasFraction(n * 10 ** count) && isFinite(10 ** count))
count++;
return count;
}
exports.decimalPlaces = decimalPlaces;
function asNumberIfValid(val) {
if (val === undefined || val === null) {
return;
}
var asNumber = Number(val);
if (isNaN(asNumber) || isFinite(asNumber) === false) {
return;
}
if (asNumber === 0) {
return;
}
return asNumber;
}
exports.asNumberIfValid = asNumberIfValid;
function upperCaseSymbols(symbols) {
if (symbols !== undefined) {
return symbols.map((s) => s.toUpperCase());
}
return;
}
exports.upperCaseSymbols = upperCaseSymbols;
function lowerCaseSymbols(symbols) {
if (symbols !== undefined) {
return symbols.map((s) => s.toLowerCase());
}
return;
}
exports.lowerCaseSymbols = lowerCaseSymbols;
const fromMicroSecondsToDate = (micros) => {
const isMicroseconds = micros > 1e15; // Check if the number is likely in microseconds
if (!isMicroseconds) {
return new Date(micros);
}
const timestamp = new Date(micros / 1000);
timestamp.μs = micros % 1000;
return timestamp;
};
exports.fromMicroSecondsToDate = fromMicroSecondsToDate;
//# sourceMappingURL=handy.js.map