tardis-dev
Version:
Convenient access to tick-level historical and real-time cryptocurrency market data via Node.js
361 lines • 17.8 kB
JavaScript
import { batch, getJSON, wait } from "../handy.js";
import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase } from "./realtimefeed.js";
const binanceHttpOptions = {
timeout: 10 * 1000,
retry: {
limit: 10,
statusCodes: [418, 429, 500, 403],
maxRetryAfter: 120 * 1000
}
};
const DEFAULT_OPEN_INTEREST_MIN_AVAILABLE_WEIGHT_BUFFER = 100;
const DEFAULT_OPEN_INTEREST_POLLING_INTERVAL_MS = 5 * 1000;
const OPEN_INTEREST_BATCH_SIZE = 10;
const OPEN_INTEREST_REQUEST_WEIGHT = 1;
const OPEN_INTEREST_POLLING_RECOVERY_MS = 1000;
const OPEN_INTEREST_MAX_POLLING_INTERVAL_MS = 60 * 1000;
const BINANCE_FUTURES_PUBLIC_CHANNELS = new Set(['bookTicker', 'depth', 'depthSnapshot', 'trade']);
const BINANCE_FUTURES_DEFAULT_WS_BASE_URL = 'wss://fstream.binance.com';
const BINANCE_FUTURES_PUBLIC_STREAM_PATH = '/public/stream';
const BINANCE_FUTURES_MARKET_STREAM_PATH = '/market/stream';
function parseBinanceWeightHeader(headerValue) {
if (headerValue === undefined) {
return undefined;
}
const parsed = Number.parseInt(headerValue, 10);
return Number.isFinite(parsed) ? parsed : undefined;
}
function getExchangeScopedNumberEnv(exchange, suffix, fallback) {
const envName = `${exchange.toUpperCase().replace(/-/g, '_')}_${suffix}`;
const rawValue = process.env[envName];
if (rawValue === undefined) {
return fallback;
}
const parsed = Number.parseInt(rawValue, 10);
return Number.isFinite(parsed) ? parsed : fallback;
}
function getExchangeScopedWssUrlEnv(exchange) {
const envName = `WSS_URL_${exchange.toUpperCase().replace(/-/g, '_')}`;
return process.env[envName];
}
function normalizeBinanceSplitWsBaseUrl(wssUrl) {
return wssUrl
.replace(/\/(public|market|private)\/stream$/u, '')
.replace(/\/stream$/u, '')
.replace(/\/(public|market|private)$/u, '');
}
function getBinanceFuturesWebSocketUrl(exchange, streamPath) {
const configuredWssUrl = getExchangeScopedWssUrlEnv(exchange) ?? BINANCE_FUTURES_DEFAULT_WS_BASE_URL;
const normalizedBaseUrl = normalizeBinanceSplitWsBaseUrl(configuredWssUrl);
return `${normalizedBaseUrl}${streamPath}`;
}
function getBinanceRequestWeightLimit(exchange, exchangeInfo) {
const configuredLimit = getExchangeScopedNumberEnv(exchange, 'REQUEST_WEIGHT_LIMIT', 0);
if (configuredLimit > 0) {
return configuredLimit;
}
const requestWeightLimit = exchangeInfo.rateLimits.find((d) => d.rateLimitType === 'REQUEST_WEIGHT')?.limit;
if (!requestWeightLimit) {
throw new Error('Failed to determine Binance REQUEST_WEIGHT limit');
}
return requestWeightLimit;
}
function getBinanceAvailableWeight(weightLimit, usedWeight, buffer) {
return weightLimit > 0 ? weightLimit - usedWeight - buffer : Infinity;
}
function getDelayToNextMinuteMS() {
const now = new Date();
return Math.max((61 - now.getUTCSeconds()) * 1000 - now.getUTCMilliseconds(), 1);
}
class BinanceRealTimeFeedBase extends MultiConnectionRealTimeFeedBase {
*_getRealTimeFeeds(exchange, filters, timeoutIntervalMS, onError) {
const wsFilters = filters.filter((f) => f.channel !== 'openInterest' && f.channel !== 'recentTrades' && f.channel !== 'fundingInfo' && f.channel !== 'insuranceBalance');
if (wsFilters.length > 0) {
yield new BinanceSingleConnectionRealTimeFeed(exchange, wsFilters, this.wssURL, this.httpURL, this.suffixes, this.depthRequestRequestWeight, timeoutIntervalMS, onError);
}
const openInterestFilters = filters.filter((f) => f.channel === 'openInterest');
if (openInterestFilters.length > 0) {
const instruments = openInterestFilters.flatMap((s) => s.symbols);
yield new BinanceFuturesOpenInterestClient(exchange, this.httpURL, instruments, onError);
}
}
}
class BinanceFuturesOpenInterestClient extends PoolingClientBase {
_exchange;
_httpURL;
_instruments;
_minPollingIntervalMS;
_minAvailableWeightBuffer;
_maxPollingIntervalMS;
_currentPollingIntervalMS;
_requestWeightLimit;
_usedWeight = 0;
constructor(_exchange, _httpURL, _instruments, onError) {
const minPollingIntervalMS = Math.max(getExchangeScopedNumberEnv(_exchange, 'OPEN_INTEREST_POLLING_INTERVAL_MS', DEFAULT_OPEN_INTEREST_POLLING_INTERVAL_MS), 1000);
super(_exchange, minPollingIntervalMS / 1000, onError);
this._exchange = _exchange;
this._httpURL = _httpURL;
this._instruments = _instruments;
this._minPollingIntervalMS = minPollingIntervalMS;
this._maxPollingIntervalMS = Math.max(this._minPollingIntervalMS, OPEN_INTEREST_MAX_POLLING_INTERVAL_MS);
this._currentPollingIntervalMS = minPollingIntervalMS;
this._requestWeightLimit = getExchangeScopedNumberEnv(_exchange, 'REQUEST_WEIGHT_LIMIT', 0);
this._minAvailableWeightBuffer = getExchangeScopedNumberEnv(_exchange, 'MIN_AVAILABLE_WEIGHT_BUFFER', DEFAULT_OPEN_INTEREST_MIN_AVAILABLE_WEIGHT_BUFFER);
}
getPoolingDelayMS() {
return this._currentPollingIntervalMS;
}
async poolDataToStream(outputStream) {
let waitedForRateLimit = false;
if (!this._requestWeightLimit) {
await this._initializeRateLimitInfo();
}
for (let index = 0; index < this._instruments.length;) {
if (outputStream.destroyed) {
return;
}
waitedForRateLimit = (await this._waitForAvailableWeight()) || waitedForRateLimit;
const batchSize = this._getBatchSize();
if (batchSize <= 0) {
break;
}
const instrumentsBatch = this._instruments.slice(index, index + batchSize);
index += instrumentsBatch.length;
const results = await Promise.allSettled(instrumentsBatch.map(async (instrument) => {
const openInterestResponse = await getJSON(`${this._httpURL}/openInterest?symbol=${instrument.toUpperCase()}`, binanceHttpOptions);
return {
instrument,
usedWeight: parseBinanceWeightHeader(openInterestResponse.headers['x-mbx-used-weight-1m']),
data: openInterestResponse.data
};
}));
let maxUsedWeight;
let fulfilledCount = 0;
for (const result of results) {
if (result.status === 'rejected') {
this._notifyError(result.reason);
continue;
}
fulfilledCount++;
maxUsedWeight = Math.max(maxUsedWeight ?? 0, result.value.usedWeight ?? 0);
if (outputStream.writable) {
outputStream.write({
stream: `${result.value.instrument.toLowerCase()}`,
generated: true,
data: result.value.data
});
}
}
this._updateUsedWeight(maxUsedWeight || undefined, fulfilledCount * OPEN_INTEREST_REQUEST_WEIGHT);
}
if (waitedForRateLimit) {
this._currentPollingIntervalMS = Math.min(this._currentPollingIntervalMS + this._minPollingIntervalMS, this._maxPollingIntervalMS);
}
else {
this._currentPollingIntervalMS = Math.max(this._minPollingIntervalMS, this._currentPollingIntervalMS - OPEN_INTEREST_POLLING_RECOVERY_MS);
}
}
async _waitForAvailableWeight() {
const available = getBinanceAvailableWeight(this._requestWeightLimit, this._usedWeight, this._minAvailableWeightBuffer);
if (available >= OPEN_INTEREST_REQUEST_WEIGHT) {
return false;
}
const delayMS = getDelayToNextMinuteMS();
this.debug('open interest reached rate limit (limit: %s, used: %s, minimum available buffer: %s), waiting %s ms', this._requestWeightLimit, this._usedWeight, this._minAvailableWeightBuffer, delayMS);
await wait(delayMS);
// Binance request weight is tracked in a rolling 1-minute window. After waiting for the next minute
// we can resume and let the next REST response header refresh the exact current usage.
this._usedWeight = 0;
return true;
}
async _initializeRateLimitInfo() {
const exchangeInfoResponse = await getJSON(`${this._httpURL}/exchangeInfo`, binanceHttpOptions);
const exchangeInfo = exchangeInfoResponse.data;
this._requestWeightLimit = getBinanceRequestWeightLimit(this._exchange, exchangeInfo);
this._updateUsedWeight(parseBinanceWeightHeader(exchangeInfoResponse.headers['x-mbx-used-weight-1m']), OPEN_INTEREST_REQUEST_WEIGHT);
}
_getBatchSize() {
const available = getBinanceAvailableWeight(this._requestWeightLimit, this._usedWeight, this._minAvailableWeightBuffer);
return Math.min(OPEN_INTEREST_BATCH_SIZE, Math.max(0, Math.floor(available)));
}
_notifyError(error) {
const normalizedError = error instanceof Error ? error : new Error(String(error));
this.debug('open interest request error %o', normalizedError);
if (this.onError !== undefined) {
this.onError(normalizedError);
}
}
_updateUsedWeight(usedWeight, fallbackIncrement = OPEN_INTEREST_REQUEST_WEIGHT) {
if (usedWeight !== undefined) {
this._usedWeight = usedWeight;
return;
}
if (this._requestWeightLimit > 0 && fallbackIncrement > 0) {
this._usedWeight += fallbackIncrement;
}
}
}
class BinanceSingleConnectionRealTimeFeed extends RealTimeFeedBase {
wssURL;
_httpURL;
_suffixes;
_depthRequestRequestWeight;
constructor(exchange, filters, wssURL, _httpURL, _suffixes, _depthRequestRequestWeight, timeoutIntervalMS, onError) {
super(exchange, filters, timeoutIntervalMS, onError);
this.wssURL = wssURL;
this._httpURL = _httpURL;
this._suffixes = _suffixes;
this._depthRequestRequestWeight = _depthRequestRequestWeight;
}
mapToSubscribeMessages(filters) {
const payload = filters
.filter((f) => f.channel !== 'depthSnapshot')
.map((filter, index) => {
if (!filter.symbols || filter.symbols.length === 0) {
throw new Error('BinanceRealTimeFeed requires explicitly specified symbols when subscribing to live feed');
}
const suffix = this._suffixes[filter.channel];
const channel = suffix !== undefined ? `${filter.channel}@${suffix}` : filter.channel;
return {
method: 'SUBSCRIBE',
params: filter.symbols.map((symbol) => `${symbol}@${channel}`),
id: index + 1
};
});
return payload;
}
messageIsError(message) {
// subscription confirmation message
if (message.result === null) {
return false;
}
if (message.stream === undefined) {
return true;
}
if (message.error !== undefined) {
return true;
}
return false;
}
async provideManualSnapshots(filters, shouldCancel) {
const depthSnapshotFilter = filters.find((f) => f.channel === 'depthSnapshot');
if (!depthSnapshotFilter) {
return;
}
const exchangeInfoResponse = await getJSON(`${this._httpURL}/exchangeInfo`, binanceHttpOptions);
const exchangeInfo = exchangeInfoResponse.data;
const DELAY_ENV = `${this._exchange.toUpperCase().replace(/-/g, '_')}_SNAPSHOTS_DELAY_MS`;
const currentWeightLimit = getBinanceRequestWeightLimit(this._exchange, exchangeInfo);
let usedWeight = Number.parseInt(exchangeInfoResponse.headers['x-mbx-used-weight-1m']);
this.debug('current x-mbx-used-weight-1m limit: %s, already used weight: %s', currentWeightLimit, usedWeight);
let concurrencyLimit = 4;
const CONCURRENCY_LIMIT_WEIGHT_ENV = `${this._exchange.toUpperCase().replace(/-/g, '_')}_CONCURRENCY_LIMIT`;
if (process.env[CONCURRENCY_LIMIT_WEIGHT_ENV] !== undefined) {
concurrencyLimit = Number.parseInt(process.env[CONCURRENCY_LIMIT_WEIGHT_ENV]);
}
this.debug('current snapshots requests concurrency limit: %s', concurrencyLimit);
const minWeightBuffer = getExchangeScopedNumberEnv(this._exchange, 'MIN_AVAILABLE_WEIGHT_BUFFER', 2 * concurrencyLimit * this._depthRequestRequestWeight);
for (const symbolsBatch of batch(depthSnapshotFilter.symbols, concurrencyLimit)) {
if (shouldCancel()) {
return;
}
this.debug('requesting manual snapshots for: %s', symbolsBatch);
const usedWeights = await Promise.all(symbolsBatch.map(async (symbol) => {
if (shouldCancel()) {
return 0;
}
const isOverRateLimit = getBinanceAvailableWeight(currentWeightLimit, usedWeight, minWeightBuffer) < 0;
if (isOverRateLimit) {
const delayMS = getDelayToNextMinuteMS();
this.debug('reached rate limit (x-mbx-used-weight-1m limit: %s, used weight: %s, minimum available weight buffer: %s), waiting: %s seconds', currentWeightLimit, usedWeight, minWeightBuffer, Math.ceil(delayMS / 1000));
await wait(delayMS);
}
const depthSnapshotResponse = await getJSON(`${this._httpURL}/depth?symbol=${symbol.toUpperCase()}&limit=1000`, binanceHttpOptions);
const snapshot = {
stream: `${symbol}`,
generated: true,
data: depthSnapshotResponse.data
};
this.manualSnapshotsBuffer.push(snapshot);
if (process.env[DELAY_ENV] !== undefined) {
const msToWait = Number.parseInt(process.env[DELAY_ENV]);
await wait(msToWait);
}
return Number.parseInt(depthSnapshotResponse.headers['x-mbx-used-weight-1m']);
}));
usedWeight = Math.max(...usedWeights);
this.debug('requested manual snapshots successfully for: %s, used weight: %s', symbolsBatch, usedWeight);
}
this.debug('requested all manual snapshots successfully');
}
}
class BinanceFuturesSingleConnectionRealTimeFeed extends BinanceSingleConnectionRealTimeFeed {
_streamPath;
constructor(exchange, filters, _streamPath, httpURL, suffixes, depthRequestRequestWeight, timeoutIntervalMS, onError) {
super(exchange, filters, getBinanceFuturesWebSocketUrl(exchange, _streamPath), httpURL, suffixes, depthRequestRequestWeight, timeoutIntervalMS, onError);
this._streamPath = _streamPath;
}
async getWebSocketUrl() {
return getBinanceFuturesWebSocketUrl(this._exchange, this._streamPath);
}
}
export class BinanceRealTimeFeed extends BinanceRealTimeFeedBase {
wssURL = 'wss://stream.binance.com/stream?timeUnit=microsecond';
httpURL = 'https://api.binance.com/api/v1';
suffixes = {
depth: '100ms'
};
depthRequestRequestWeight = 10;
}
export class BinanceJerseyRealTimeFeed extends BinanceRealTimeFeedBase {
wssURL = 'wss://stream.binance.je:9443/stream';
httpURL = 'https://api.binance.je/api/v1';
suffixes = {
depth: '100ms'
};
depthRequestRequestWeight = 10;
}
export class BinanceUSRealTimeFeed extends BinanceRealTimeFeedBase {
wssURL = 'wss://stream.binance.us:9443/stream';
httpURL = 'https://api.binance.us/api/v1';
suffixes = {
depth: '100ms'
};
depthRequestRequestWeight = 10;
}
export class BinanceFuturesRealTimeFeed extends BinanceRealTimeFeedBase {
wssURL = `${BINANCE_FUTURES_DEFAULT_WS_BASE_URL}${BINANCE_FUTURES_PUBLIC_STREAM_PATH}`;
httpURL = 'https://fapi.binance.com/fapi/v1';
suffixes = {
depth: '0ms',
markPrice: '1s'
};
depthRequestRequestWeight = 20;
*_getRealTimeFeeds(exchange, filters, timeoutIntervalMS, onError) {
const wsFilters = filters.filter((f) => f.channel !== 'openInterest' && f.channel !== 'recentTrades' && f.channel !== 'fundingInfo' && f.channel !== 'insuranceBalance');
const publicWsFilters = wsFilters.filter((f) => BINANCE_FUTURES_PUBLIC_CHANNELS.has(f.channel));
if (publicWsFilters.length > 0) {
yield new BinanceFuturesSingleConnectionRealTimeFeed(exchange, publicWsFilters, BINANCE_FUTURES_PUBLIC_STREAM_PATH, this.httpURL, this.suffixes, this.depthRequestRequestWeight, timeoutIntervalMS, onError);
}
const marketWsFilters = wsFilters.filter((f) => BINANCE_FUTURES_PUBLIC_CHANNELS.has(f.channel) === false);
if (marketWsFilters.length > 0) {
yield new BinanceFuturesSingleConnectionRealTimeFeed(exchange, marketWsFilters, BINANCE_FUTURES_MARKET_STREAM_PATH, this.httpURL, this.suffixes, this.depthRequestRequestWeight, timeoutIntervalMS, onError);
}
const openInterestFilters = filters.filter((f) => f.channel === 'openInterest');
if (openInterestFilters.length > 0) {
const instruments = openInterestFilters.flatMap((s) => s.symbols);
yield new BinanceFuturesOpenInterestClient(exchange, this.httpURL, instruments, onError);
}
}
}
export class BinanceDeliveryRealTimeFeed extends BinanceRealTimeFeedBase {
wssURL = 'wss://dstream.binance.com/stream';
httpURL = 'https://dapi.binance.com/dapi/v1';
suffixes = {
depth: '0ms',
markPrice: '1s',
indexPrice: '1s'
};
depthRequestRequestWeight = 20;
}
//# sourceMappingURL=binance.js.map