tardis-dev
Version:
Convenient access to tick-level historical and real-time cryptocurrency market data via Node.js
672 lines • 25 kB
JavaScript
import crypto, { createHash } from 'crypto';
import { createWriteStream, mkdirSync, rmSync } from 'node:fs';
import { rename } from 'node:fs/promises';
import followRedirects from 'follow-redirects';
import * as httpsProxyAgentPkg from 'https-proxy-agent';
import path from 'path';
import { debug } from "./debug.js";
import * as socksProxyAgentPkg from 'socks-proxy-agent';
const { http, https } = followRedirects;
const { HttpsProxyAgent } = httpsProxyAgentPkg;
const { SocksProxyAgent } = socksProxyAgentPkg;
export function parseAsUTCDate(val) {
// Treat date-only and minute-level strings as UTC instead of local time.
if (val.endsWith('Z') === false) {
val += 'Z';
}
const date = new Date(val);
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours(), date.getUTCMinutes()));
}
export function wait(delayMS) {
return new Promise((resolve) => {
setTimeout(resolve, delayMS);
});
}
export function getRandomString() {
return crypto.randomBytes(24).toString('hex');
}
export function formatDateToPath(date) {
const year = date.getUTCFullYear();
const month = doubleDigit(date.getUTCMonth() + 1);
const day = doubleDigit(date.getUTCDate());
const hour = doubleDigit(date.getUTCHours());
const minute = doubleDigit(date.getUTCMinutes());
return `${year}/${month}/${day}/${hour}/${minute}`;
}
export function doubleDigit(input) {
return input < 10 ? '0' + input : '' + input;
}
export function sha256(obj) {
return createHash('sha256').update(JSON.stringify(obj)).digest('hex');
}
export function addMinutes(date, minutes) {
return new Date(date.getTime() + minutes * 60000);
}
export function addDays(date, days) {
return new Date(date.getTime() + days * 60000 * 1440);
}
export function* sequence(end, seed = 0) {
let current = seed;
while (current < end) {
yield current;
current += 1;
}
return;
}
export const ONE_SEC_IN_MS = 1000;
export class HttpError extends Error {
status;
responseText;
url;
constructor(status, responseText, url) {
super(`HttpError: status code: ${status}, response text: ${responseText}`);
this.status = status;
this.responseText = responseText;
this.url = url;
}
}
class HttpClientError extends Error {
response;
method;
url;
constructor(response, method, url) {
super(`HTTP ${method} ${url} failed with status ${response.statusCode}`);
this.response = response;
this.method = method;
this.url = url;
}
}
export function* take(iterable, length) {
if (length === 0) {
return;
}
for (const item of iterable) {
yield item;
length--;
if (length === 0) {
return;
}
}
}
export async function* normalizeMessages(exchange, symbols, messages, mappers, createMappers, withDisconnectMessages, filter, currentTimestamp) {
let previousLocalTimestamp = currentTimestamp;
let mappersForExchange = mappers;
if (mappersForExchange.length === 0) {
throw new Error(`Can't normalize data without any normalizers provided`);
}
for await (const messageWithTimestamp of messages) {
if (messageWithTimestamp === undefined) {
// we received undefined meaning Websocket disconnection
// lets create new mappers with clean state for 'new connection'
mappersForExchange = undefined;
// if flag withDisconnectMessages is set, yield disconnect message
if (withDisconnectMessages === true && previousLocalTimestamp !== undefined) {
const disconnect = {
type: 'disconnect',
exchange,
localTimestamp: previousLocalTimestamp,
symbols
};
yield disconnect;
}
continue;
}
if (mappersForExchange === undefined) {
mappersForExchange = createMappers(messageWithTimestamp.localTimestamp);
}
previousLocalTimestamp = messageWithTimestamp.localTimestamp;
for (const mapper of mappersForExchange) {
if (mapper.canHandle(messageWithTimestamp.message)) {
const mappedMessages = mapper.map(messageWithTimestamp.message, messageWithTimestamp.localTimestamp);
if (!mappedMessages) {
continue;
}
for (const message of mappedMessages) {
if (filter === undefined) {
yield message;
}
else if (filter(message.symbol)) {
yield message;
}
}
}
}
}
}
export function getFilters(mappers, symbols) {
const filters = mappers.flatMap((mapper) => mapper.getFilters(symbols));
const deduplicatedFilters = filters.reduce((prev, current) => {
const matchingExisting = prev.find((c) => c.channel === current.channel);
if (matchingExisting !== undefined) {
if (matchingExisting.symbols !== undefined && current.symbols) {
for (let symbol of current.symbols) {
if (matchingExisting.symbols.includes(symbol) === false) {
matchingExisting.symbols.push(symbol);
}
}
}
else if (current.symbols) {
matchingExisting.symbols = [...current.symbols];
}
}
else {
prev.push(current);
}
return prev;
}, []);
return deduplicatedFilters;
}
export function* batch(symbols, batchSize) {
for (let i = 0; i < symbols.length; i += batchSize) {
yield symbols.slice(i, i + batchSize);
}
}
export function* batchObjects(payload, batchSize) {
for (let i = 0; i < payload.length; i += batchSize) {
yield payload.slice(i, i + batchSize);
}
}
export function parseμs(dateString) {
// check if we have ISO 8601 format date string, e.g: 2019-06-01T00:03:03.1238784Z or 2020-07-22T00:09:16.836773Z
// or 2020-03-01T00:00:24.893456+00:00
if (dateString.length === 27 || dateString.length === 28 || dateString.length === 32 || dateString.length === 30) {
return Number(dateString.slice(23, 26));
}
return 0;
}
export function optimizeFilters(filters) {
// deduplicate filters (if the channel was provided multiple times)
const optimizedFilters = filters.reduce((prev, current) => {
const matchingExisting = prev.find((c) => c.channel === current.channel);
if (matchingExisting) {
// both previous and current have symbols let's merge them
if (matchingExisting.symbols && current.symbols) {
matchingExisting.symbols.push(...current.symbols);
}
else if (current.symbols) {
matchingExisting.symbols = [...current.symbols];
}
}
else {
prev.push(current);
}
return prev;
}, []);
// sort filters in place to improve local disk cache ratio (no matter filters order if the same filters are provided will hit the cache)
optimizedFilters.sort((f1, f2) => {
if (f1.channel < f2.channel) {
return -1;
}
if (f1.channel > f2.channel) {
return 1;
}
return 0;
});
// sort and deduplicate filters symbols
optimizedFilters.forEach((filter) => {
if (filter.symbols) {
filter.symbols = [...new Set(filter.symbols)].sort();
}
});
return optimizedFilters;
}
const httpsAgent = new https.Agent({
keepAlive: true,
keepAliveMsecs: 10 * ONE_SEC_IN_MS,
maxSockets: 120
});
export const httpsProxyAgent = process.env.HTTP_PROXY !== undefined
? new HttpsProxyAgent(process.env.HTTP_PROXY)
: process.env.SOCKS_PROXY !== undefined
? new SocksProxyAgent(process.env.SOCKS_PROXY)
: undefined;
const DEFAULT_FETCH_RETRY_LIMIT = 2;
function getRetrySettings(method, retry) {
const retryOptions = typeof retry === 'object' ? retry : undefined;
const retryEnabled = method === 'GET' || retry !== undefined;
const limit = typeof retry === 'number' ? retry : (retryOptions?.limit ?? (retryEnabled ? DEFAULT_FETCH_RETRY_LIMIT : 0));
return {
limit,
maxRetryAfter: retryOptions?.maxRetryAfter,
statusCodes: retryOptions?.statusCodes ? new Set(retryOptions.statusCodes) : undefined
};
}
function parseResponseHeaders(headers) {
return Object.fromEntries(headers.entries());
}
function parseNodeResponseHeaders(headers) {
return Object.fromEntries(Object.entries(headers).flatMap(([key, value]) => {
if (value === undefined) {
return [];
}
return [[key.toLowerCase(), Array.isArray(value) ? value.join(', ') : value]];
}));
}
function createHttpResponse(statusCode, headers, body) {
return {
statusCode,
headers,
body
};
}
function prepareRequest(method, options) {
if (options.body === undefined) {
return {
headers: options.headers,
body: undefined
};
}
const headers = { ...options.headers };
const body = typeof options.body === 'string' ? options.body : JSON.stringify(options.body);
if (method !== 'GET' && headers['Content-Type'] === undefined && headers['content-type'] === undefined) {
headers['Content-Type'] = 'application/json';
}
return {
headers,
body
};
}
function getRetryAfterDelayMS(headers, maxRetryAfter) {
const retryAfterHeader = headers['retry-after'];
if (retryAfterHeader === undefined) {
return;
}
const parsedSeconds = Number.parseFloat(retryAfterHeader);
let delayMS;
if (Number.isFinite(parsedSeconds)) {
delayMS = parsedSeconds * ONE_SEC_IN_MS;
}
else {
const parsedDate = Date.parse(retryAfterHeader);
if (Number.isFinite(parsedDate)) {
delayMS = parsedDate - Date.now();
}
}
if (delayMS === undefined || delayMS < 0) {
return;
}
if (maxRetryAfter !== undefined && delayMS > maxRetryAfter) {
return;
}
return delayMS;
}
function getRetryDelayMS(attempt, headers, maxRetryAfter) {
const retryAfterDelayMS = getRetryAfterDelayMS(headers, maxRetryAfter);
if (retryAfterDelayMS !== undefined) {
return retryAfterDelayMS;
}
return Math.min(250 * 2 ** (attempt - 1), 5000);
}
function isRetryableStatus(statusCode, retrySettings) {
if (retrySettings.statusCodes !== undefined) {
return retrySettings.statusCodes.has(statusCode);
}
return statusCode === 408 || statusCode === 429 || statusCode >= 500;
}
function shouldRetryHttpStatus(attempt, response, retrySettings) {
return attempt <= retrySettings.limit && isRetryableStatus(response.statusCode, retrySettings);
}
function shouldRetryHttpError(attempt, retrySettings) {
return attempt <= retrySettings.limit;
}
async function requestViaFetch(method, url, options) {
const controller = new AbortController();
const timeoutMS = options.timeout;
const timeoutId = timeoutMS !== undefined ? setTimeout(() => controller.abort(), timeoutMS) : undefined;
const preparedRequest = prepareRequest(method, options);
try {
const response = await fetch(url, {
method,
headers: preparedRequest.headers,
body: preparedRequest.body,
signal: controller.signal
});
const body = await response.text();
return createHttpResponse(response.status, parseResponseHeaders(response.headers), body);
}
catch (error) {
if (controller.signal.aborted) {
throw new Error('Request timed out');
}
throw error;
}
finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
}
async function requestViaProxy(method, url, options) {
const requestClient = new URL(url).protocol === 'http:' ? http : https;
const preparedRequest = prepareRequest(method, options);
return await new Promise((resolve, reject) => {
const request = requestClient
.request(url, {
method,
agent: httpsProxyAgent,
headers: preparedRequest.headers,
timeout: options.timeout
}, (response) => {
response.setEncoding('utf8');
let body = '';
response.on('error', reject);
response.on('data', (chunk) => (body += chunk));
response.on('end', () => {
resolve(createHttpResponse(response.statusCode ?? 0, parseNodeResponseHeaders(response.headers), body));
});
})
.on('error', reject)
.on('timeout', () => {
reject(new Error('Request timed out'));
request.destroy();
});
if (preparedRequest.body !== undefined) {
request.write(preparedRequest.body);
}
request.end();
});
}
async function request(method, url, options = {}) {
const retrySettings = getRetrySettings(method, options.retry);
for (let attempt = 1;; attempt += 1) {
try {
const response = httpsProxyAgent === undefined ? await requestViaFetch(method, url, options) : await requestViaProxy(method, url, options);
if (response.statusCode >= 200 && response.statusCode < 300) {
return response;
}
if (shouldRetryHttpStatus(attempt, response, retrySettings)) {
await wait(getRetryDelayMS(attempt, response.headers, retrySettings.maxRetryAfter));
continue;
}
throw new HttpClientError(response, method, url);
}
catch (error) {
if (error instanceof HttpClientError) {
throw error;
}
if (shouldRetryHttpError(attempt, retrySettings)) {
await wait(Math.min(250 * 2 ** (attempt - 1), 5000));
continue;
}
throw error;
}
}
}
async function requestJSON(method, url, options) {
const response = await request(method, url, options);
return {
data: JSON.parse(response.body),
headers: response.headers,
statusCode: response.statusCode
};
}
export function getJSON(url, options) {
return requestJSON('GET', url, options);
}
export function postJSON(url, options) {
return requestJSON('POST', url, options);
}
export async function download({ apiKey, downloadPath, url, userAgent, appendContentEncodingExtension = false, acceptEncoding = 'gzip' }) {
const httpRequestOptions = {
agent: httpsProxyAgent !== undefined ? httpsProxyAgent : httpsAgent,
timeout: 90 * ONE_SEC_IN_MS,
headers: {
'Accept-Encoding': acceptEncoding,
'User-Agent': userAgent,
Authorization: apiKey ? `Bearer ${apiKey}` : ''
}
};
const MAX_ATTEMPTS = 30;
let attempts = 0;
while (true) {
// simple retry logic when fetching from the network...
attempts++;
try {
const addRetryAttempt = attempts - 1 > 0 && url.endsWith('gz');
if (addRetryAttempt) {
return await _downloadFile(httpRequestOptions, `${url}?retryAttempt=${attempts - 1}`, downloadPath, appendContentEncodingExtension);
}
else {
return await _downloadFile(httpRequestOptions, url, downloadPath, appendContentEncodingExtension);
}
}
catch (error) {
const unsupportedDataFeedEncoding = error instanceof Error && error.message.startsWith('Unsupported data feed content encoding');
const badOrUnauthorizedRequest = error instanceof HttpError &&
((error.status === 400 && error.message.includes('ISO 8601 format') === false) || error.status === 401);
const tooManyRequests = error instanceof HttpError && error.status === 429;
const internalServiceError = error instanceof HttpError && error.status === 500;
// do not retry when we've got bad or unauthorized request or enough attempts
if (unsupportedDataFeedEncoding || badOrUnauthorizedRequest || attempts === MAX_ATTEMPTS) {
throw error;
}
const randomIngridient = Math.random() * 500;
const attemptsDelayMS = Math.min(Math.pow(2, attempts) * ONE_SEC_IN_MS, 120 * ONE_SEC_IN_MS);
let nextAttemptDelayMS = randomIngridient + attemptsDelayMS;
if (tooManyRequests) {
// when too many requests received wait one minute
nextAttemptDelayMS += 60 * ONE_SEC_IN_MS;
}
if (internalServiceError) {
nextAttemptDelayMS = nextAttemptDelayMS * 2;
}
debug('download file error: %o, next attempt delay: %d, url %s, path: %s', error, nextAttemptDelayMS, url, downloadPath);
await wait(nextAttemptDelayMS);
}
}
}
const tmpFileCleanups = new Map();
export function cleanTempFiles() {
tmpFileCleanups.forEach((cleanup) => cleanup());
}
async function _downloadFile(requestOptions, url, downloadPath, appendContentEncodingExtension) {
// first ensure that directory where we want to download file exists
mkdirSync(path.dirname(downloadPath), { recursive: true });
// create write file stream that we'll write data into - first as unconfirmed temp file
const tmpFilePath = `${downloadPath}${crypto.randomBytes(8).toString('hex')}.unconfirmed`;
const fileWriteStream = createWriteStream(tmpFilePath);
const cleanup = () => {
try {
fileWriteStream.destroy();
rmSync(tmpFilePath, { force: true });
}
catch { }
};
tmpFileCleanups.set(tmpFilePath, cleanup);
let finalDownloadPath = downloadPath;
try {
// based on https://github.com/nodejs/node/issues/28172 - only reliable way to consume response stream and avoiding all the 'gotchas'
let responseHeaders = {};
await new Promise((resolve, reject) => {
const req = https
.get(url, requestOptions, (res) => {
const { statusCode } = res;
if (statusCode !== 200) {
// read the error response text and throw it as an HttpError
res.setEncoding('utf8');
let body = '';
res.on('error', reject);
res.on('data', (chunk) => (body += chunk));
res.on('end', () => {
reject(new HttpError(statusCode, body, url));
});
}
else {
responseHeaders = parseNodeResponseHeaders(res.headers);
if (appendContentEncodingExtension) {
const contentEncoding = asSingleHeaderValue(res.headers['content-encoding']);
if (contentEncoding === 'zstd') {
finalDownloadPath = `${downloadPath}.zst`;
}
else if (contentEncoding === undefined || contentEncoding === 'gzip') {
finalDownloadPath = `${downloadPath}.gz`;
}
else {
reject(new Error(`Unsupported data feed content encoding: ${contentEncoding}`));
return;
}
}
// consume the response stream by writing it to the file
res
.on('error', reject)
.on('aborted', () => reject(new Error('Request aborted')))
.pipe(fileWriteStream)
.on('error', reject)
.on('finish', () => {
if (res.complete) {
resolve();
}
else {
reject(new Error('The connection was terminated while the message was still being sent'));
}
});
}
})
.on('error', reject)
.on('timeout', () => {
debug('download file request timeout, %s', url);
reject(new Error('Request timed out'));
req.destroy();
});
});
// finally when saving from the network to file has succeded, rename tmp file to normal name
// then we're sure that responses is 100% saved and also even if different process was doing the same we're good
await rename(tmpFilePath, finalDownloadPath);
return {
downloadPath: finalDownloadPath,
headers: responseHeaders
};
}
finally {
tmpFileCleanups.delete(tmpFilePath);
cleanup();
}
}
function asSingleHeaderValue(headerValue) {
if (Array.isArray(headerValue)) {
return headerValue[0];
}
return headerValue;
}
export class CircularBuffer {
_bufferSize;
_buffer = [];
_index = 0;
constructor(_bufferSize) {
this._bufferSize = _bufferSize;
}
append(value) {
const isFull = this._buffer.length === this._bufferSize;
let poppedValue;
if (isFull) {
poppedValue = this._buffer[this._index];
}
this._buffer[this._index] = value;
this._index = (this._index + 1) % this._bufferSize;
return poppedValue;
}
*items() {
for (let i = 0; i < this._buffer.length; i++) {
const index = (this._index + i) % this._buffer.length;
yield this._buffer[index];
}
}
get count() {
return this._buffer.length;
}
clear() {
this._buffer = [];
this._index = 0;
}
}
export class CappedSet {
_maxSize;
_set = new Set();
constructor(_maxSize) {
this._maxSize = _maxSize;
}
has(value) {
return this._set.has(value);
}
add(value) {
if (this._set.size >= this._maxSize) {
this._set.delete(this._set.keys().next().value);
}
this._set.add(value);
}
remove(value) {
this._set.delete(value);
}
size() {
return this._set.size;
}
}
function hasFraction(n) {
return Math.abs(Math.round(n) - n) > 1e-10;
}
// https://stackoverflow.com/a/44815797
export function decimalPlaces(n) {
let count = 0;
// multiply by increasing powers of 10 until the fractional part is ~ 0
while (hasFraction(n * 10 ** count) && isFinite(10 ** count))
count++;
return count;
}
/**
* Parses optional numeric fields where:
* * `0` **is valid and preserved**
* * `undefined`, `null`, `NaN`, `Infinity`, and `-Infinity` are treated as not valid and mapped to `undefined`
*
* Use for nullable/optional numeric fields such as open interest, funding rates, and greeks.
*/
export function asNumberOrUndefined(val) {
if (val === undefined || val === null || val === '') {
return;
}
if (typeof val === 'number') {
return Number.isFinite(val) ? val : undefined;
}
const asNumber = Number(val);
return Number.isFinite(asNumber) ? asNumber : undefined;
}
/**
* Parses optional numeric fields where:
* * `0`, `undefined`, `null`, `NaN`, `Infinity`, and `-Infinity` are treated as not valid and mapped to `undefined`.
*
* Use for empty quote/top-of-book values that exchanges encode as zero.
*/
export function asNonZeroNumberOrUndefined(val) {
if (val === undefined || val === null || val === '' || val === 0) {
return;
}
if (typeof val === 'number') {
return Number.isFinite(val) ? val : undefined;
}
const asNumber = Number(val);
return Number.isFinite(asNumber) && asNumber !== 0 ? asNumber : undefined;
}
export function upperCaseSymbols(symbols) {
if (symbols !== undefined) {
return symbols.map((s) => s.toUpperCase());
}
return;
}
export function lowerCaseSymbols(symbols) {
if (symbols !== undefined) {
return symbols.map((s) => s.toLowerCase());
}
return;
}
export const fromMicroSecondsToDate = (micros) => {
const isMicroseconds = micros > 1e15; // Check if the number is likely in microseconds
if (!isMicroseconds) {
return new Date(micros);
}
const timestamp = new Date(micros / 1000);
timestamp.μs = micros % 1000;
return timestamp;
};
export function onlyUnique(value, index, array) {
return array.indexOf(value) === index;
}
//# sourceMappingURL=handy.js.map