tlab-trading-toolkit
Version:
A trading toolkit for building advanced trading bots on the GDAX platform
508 lines (507 loc) • 23.9 kB
JavaScript
'use strict';
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
/***************************************************************************************************************************
* @license *
* Copyright 2017 Coinbase, Inc. *
* *
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance *
* with the License. You may obtain a copy of the License at *
* *
* http://www.apache.org/licenses/LICENSE-2.0 *
* *
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on *
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the *
* License for the specific language governing permissions and limitations under the License. *
***************************************************************************************************************************/
const BinanceAPI_1 = require("./BinanceAPI");
const ExchangeFeed_1 = require("../ExchangeFeed");
const types_1 = require("../../lib/types");
const WebSocket = require("ws");
const request = require("request-promise");
const timers_1 = require("timers");
exports.BINANCE_WS_FEED = `wss://stream.binance.com:9443/ws/`;
// hooks for replacing libraries if desired
const hooks = {
WebSocket: WebSocket
};
var startingTime = Date.now();
var index = 0;
var underBan = false;
var lastBanRef;
var banUntilTime = 0;
var getBanTime = function (str) {
const regex = /IP banned until (\d*)./g;
let m;
var time = 0;
while ((m = regex.exec(str)) !== null) {
// This is necessary to avoid infinite loops with zero-width matches
if (m.index === regex.lastIndex) {
regex.lastIndex++;
}
// The result can be accessed through the `m`-variable.
m.forEach((match, groupIndex) => {
time = parseInt(match);
banUntilTime = time;
});
}
return time;
};
var retryCount = process.env.RETRY_COUNT || 1;
class BinanceFeed extends ExchangeFeed_1.ExchangeFeed {
constructor(config) {
super(config);
this.lastHeartBeat = -1;
this.lastMessageTime = {};
this.lastTradeTime = {};
this.counters = {};
this.sequences = {};
this.initialMessagesQueue = {};
this.depthsockets = {};
this.tradesockets = {};
this.MAX_QUEUE_LENGTH = 500;
this.erroredProducts = new Set();
this.owner = 'Binance';
this.multiSocket = true;
this.feedUrl = exports.BINANCE_WS_FEED;
this.connect(config.products);
}
getWebsocketUrlForProduct(product) {
return exports.BINANCE_WS_FEED + product.toLowerCase() + '@depth';
}
retryErroredProducts() {
console.log(' Total Errored products ', this.erroredProducts.size);
if (this.erroredProducts.size > 0) {
Array.from(this.erroredProducts).forEach(this.subscribeProduct.bind(this));
console.log('=========================================================================');
console.log('could not subscribe following products ', Array.from(this.erroredProducts));
console.log('=========================================================================');
}
else {
console.log('=========================================================================');
console.log('All products subscribed');
console.log('Subscribe completed @ ', new Date());
console.log('=========================================================================');
}
}
connect(products) {
return __awaiter(this, void 0, void 0, function* () {
console.log('Is multi sockets : ', this.multiSocket);
console.log('Products list : ', products);
if (this.isConnecting || this.isConnected()) {
return;
}
this.isConnecting = true;
timers_1.setTimeout(() => {
this.emit('websocket-connection');
}, 3000);
if (this.multiSocket && products && products.length > 0) {
for (let product of products) {
this.counters[product] = -1;
this.lastMessageTime[product] = 0;
this.initialMessagesQueue[product] = [];
yield this.subscribeProduct(product);
}
this.retryErroredProducts();
startingTime = Date.now();
}
console.log('=============================================');
console.log('Setting up heart beat checker for depth and trade every 0.4 minutes after 0.1 min');
console.log('=============================================');
timers_1.setTimeout(() => {
Object.keys(this.lastMessageTime).forEach((product) => {
var now = Date.now();
var tradeSocket = this.tradesockets[product];
var depthSocket = this.depthsockets[product];
if ((tradeSocket.readyState > 0) && (depthSocket.readyState > 0)) {
tradeSocket.ping(now);
depthSocket.ping(now);
}
});
setInterval(() => {
var now = Date.now();
console.log('Verifying depth and trade socket status and testing ping @', now);
Object.keys(this.lastMessageTime).forEach((product) => {
try {
var failed = false;
var tradeSocket = this.tradesockets[product];
var depthSocket = this.depthsockets[product];
if ((tradeSocket.readyState > 0) && (depthSocket.readyState > 0)) {
tradeSocket.ping(now);
depthSocket.ping(now);
}
var tradePong = tradeSocket.lastPongTime;
var depthPong = depthSocket.lastPongTime;
var tradePonged = tradePong > (now - (3 * 60 * 1000));
var depthPonged = depthPong > (now - (3 * 60 * 1000));
var lastReceived = this.lastMessageTime[product];
var lastTraded = this.lastTradeTime[product];
var elapsed = now - lastReceived;
var tradeElapsed = now - lastTraded;
if ((!tradePonged) || (!depthPonged) || (tradeSocket.readyState > 1) || (depthSocket.readyState > 1)) {
console.log('Product : ', product);
console.log('Current Time : ', new Date());
console.log('Trade Ponged : ', new Date(tradePong));
console.log('Depth Ponged : ', new Date(depthPong));
console.log('Last trade times : ', new Date(lastTraded));
console.log('Last Depth changed : ', new Date(lastReceived));
console.log('Elapsed : ', elapsed / 1000, 'secs');
console.log('Trade Elapsed : ', tradeElapsed / 1000, 'secs');
console.log(`
Trade Ponged : ${(!tradePonged)}
Depth Ponged : ${(!depthPonged)}
Trade Ready State : ${(tradeSocket.readyState > 1)}
Depth Ready State : ${(depthSocket.readyState > 1)}
`);
failed = true;
console.log('Socket not working for product ', product);
this.subscribeProduct(product);
}
}
catch (err) {
console.error(err);
}
});
}, 1000 * 60 * 0.6);
}, 1000 * 60 * 4);
});
}
subscribeProduct(product) {
return __awaiter(this, void 0, void 0, function* () {
try {
if (underBan) {
console.warn('Under ban not subscribing product', product);
return;
}
index++;
console.log(index);
if (index % 3 === 0) {
yield new Promise((resolve) => timers_1.setTimeout(resolve, 10500));
}
var initialTime = Date.now();
var oldTradeSocket = this.tradesockets[product];
var oldDepthSocket = this.depthsockets[product];
if (oldTradeSocket) {
oldTradeSocket.active = false;
oldTradeSocket.close();
}
if (oldDepthSocket) {
oldDepthSocket.active = false;
oldDepthSocket.close();
}
this.lastMessageTime[product] = initialTime;
this.lastTradeTime[product] = initialTime;
var depthUrl = this.getWebsocketUrlForProduct(product);
console.log('connecting to ', this.getWebsocketUrlForProduct(product));
const depthSocket = new hooks.WebSocket(depthUrl);
depthSocket.active = true;
var resolved = false;
depthSocket.on('message', (msg) => {
this.lastMessageTime[product] = Date.now();
this.handleDepthMessages(msg, product);
});
depthSocket.on('close', (data) => {
if (depthSocket.active) {
console.log('Active Depth socket closed resubscribing', product, data);
this.subscribeProduct(product);
}
else {
console.log('Inactive Depth socket closed ignoring', product, data);
}
});
depthSocket.on('error', (data) => {
if (depthSocket.active) {
console.log('Active Depth socket errored resubscribing', product, data);
this.subscribeProduct(product);
}
else {
console.log('Inactive Depth socket errored ignoring', product, data);
}
});
var depthPromise = new Promise((resolve, reject) => {
var timeout = timers_1.setTimeout(() => {
reject('TIMEDOUT');
}, 20000);
depthSocket.on('pong', (data) => {
try {
if (!resolved) {
console.log('Received pong after connect for ', product);
clearTimeout(timeout);
resolved = true;
resolve(true);
}
depthSocket.lastPongTime = parseInt(data.toString());
}
catch (err) {
}
});
});
depthSocket.on('open', () => {
depthSocket.ping(Date.now());
});
const tradesocket = new hooks.WebSocket(exports.BINANCE_WS_FEED + product.toLowerCase() + '@trade');
console.log('connecting to ', exports.BINANCE_WS_FEED + product.toLowerCase() + '@trade');
tradesocket.active = true;
tradesocket.on('message', (msg) => {
this.lastTradeTime[product] = Date.now();
this.handleTradeMessages(msg, product);
});
tradesocket.on('close', (data) => {
if (depthSocket.active) {
console.log('Active Trade socket closed resubscribing', product, data);
this.subscribeProduct(product);
}
else {
console.log('Inactive Trade socket closed ignoring', product, data);
}
});
tradesocket.on('error', (data) => {
if (depthSocket.active) {
console.log('Active Trade socket errored resubscribing', product, data);
this.subscribeProduct(product);
}
else {
console.log('Inactive Trade socket errored ignoring', product, data);
}
});
tradesocket.on('pong', (data) => {
try {
tradesocket.lastPongTime = parseInt(data.toString());
}
catch (err) {
}
});
var tradePromise = new Promise((resolve, reject) => {
var timeout = timers_1.setTimeout(() => {
reject('TIMEDOUT');
}, 20000);
tradesocket.on('open', () => {
tradesocket.ping(Date.now());
clearTimeout(timeout);
resolve(true);
});
});
this.tradesockets[product] = tradesocket;
this.depthsockets[product] = depthSocket;
depthSocket.lastPongTime = initialTime;
tradesocket.lastPongTime = initialTime;
console.log('Waiting for trade and depth socket to connect for ', product);
var result = yield depthPromise;
var result = yield tradePromise;
console.log('Connected to both trade and depth, fetching snaphsot for ', product);
this.fetchSnapshotForProduct(product);
}
catch (err) {
if (err === 'TIMEDOUT') {
this.subscribeProduct(product);
return;
}
console.warn('Error occured when subscribing for product ', product);
this.erroredProducts.add(product);
console.error(err);
}
});
}
fetchSnapshotForProduct(product) {
request(`https://www.binance.com/api/v1/depth?symbol=${product.toUpperCase()}&limit=1000`, { json: true }).then((depthSnapshot) => {
console.log('Received Snapshot ', product);
this.handleSnapshotMessage(depthSnapshot, product);
}).catch((err) => {
if (err.statusCode == 418) {
underBan = true;
var currentTime = Date.now();
var ban = getBanTime(err.message);
console.log('Removing ban @ ', ban, ' after ', (ban - currentTime) / 1000, 'secs');
clearTimeout(lastBanRef);
lastBanRef = timers_1.setTimeout(() => {
underBan = false;
this.retryErroredProducts();
}, (ban - currentTime));
}
else if (err.statusCode == 429) {
underBan = true;
clearTimeout(lastBanRef);
lastBanRef = timers_1.setTimeout(() => {
underBan = false;
console.log('Retry after 50 secs');
this.retryErroredProducts();
}, (30 * 1000));
}
this.erroredProducts.add(product);
console.warn('Error occured when fetching snapshot for product ', product);
console.warn(err.message);
console.error(err);
});
}
handleMessage() {
}
handleSnapshotMessage(msg, productId) {
var binanceMessage = msg;
binanceMessage.s = productId;
var feedData = this.initialMessagesQueue[productId][0];
if (feedData && (feedData.U > binanceMessage.lastUpdateId)) {
//Snaphshot still old;
console.log(`Snapshot still old, earliest feed counter ${feedData.U}, snapshot counter ${binanceMessage.lastUpdateId}`);
this.fetchSnapshotForProduct(productId);
return;
}
this.counters[productId] = binanceMessage.lastUpdateId + 1;
console.log('Snapshot received for product ', productId, ' last update ', binanceMessage.lastUpdateId);
let message = this.createSnapshotMessage(binanceMessage);
this.push(message);
}
handleTradeMessages(msg, productId) {
var binanceTradeMessage = JSON.parse(msg);
const message = {
type: 'trade',
productId: BinanceAPI_1.BinanceAPI.genericProduct(binanceTradeMessage.s),
time: new Date(+binanceTradeMessage.E),
tradeId: binanceTradeMessage.t.toString(),
price: binanceTradeMessage.p,
size: binanceTradeMessage.q,
side: binanceTradeMessage.m ? 'sell' : 'buy'
};
this.push(message);
}
handleDepthMessages(msg, productId) {
var binanceDepthMessage = JSON.parse(msg);
var messageQueue = this.initialMessagesQueue[productId];
if (this.counters[productId] > -1) {
//flush all the messages
let message = messageQueue.pop();
while (message) {
if (message.u <= (this.counters[productId] - 1)) {
message = messageQueue.pop();
continue;
}
else if (message.U <= this.counters[productId] && message.u >= this.counters[productId]) {
this.processLevelMessage(message);
this.counters[productId] = (message.u + 1);
message = messageQueue.pop();
}
else {
console.warn(`Queued message doenst match the request criteria for product ${productId} restarting`);
this.counters[productId] = -1;
this.subscribeProduct(productId);
return;
}
}
if (binanceDepthMessage.U > this.counters[productId]) {
console.warn(`Skipped message for product ${productId} restarting feed Expected : ${this.counters[productId]} got ${binanceDepthMessage.U}`);
this.counters[productId] = -1;
this.subscribeProduct(productId);
}
else {
this.counters[productId] = (binanceDepthMessage.u + 1);
this.processLevelMessage(binanceDepthMessage);
}
}
else if (this.initialMessagesQueue[productId].length > this.MAX_QUEUE_LENGTH) {
this.initialMessagesQueue[productId] = [];
console.warn('Max queue length reached restarting feed for ', productId);
this.counters[productId] = -1;
this.subscribeProduct(productId);
return;
}
else {
console.log("Havent received snapshot yet. Adding to queue for ", productId);
messageQueue.push(binanceDepthMessage);
}
}
nextSequence(prodcutId) {
var seq = this.sequences[prodcutId] + 1;
this.sequences[prodcutId] = seq;
return seq;
}
processLevelMessage(depthMessage) {
var genericProduct = BinanceAPI_1.BinanceAPI.genericProduct(depthMessage.s);
depthMessage.b.forEach((level) => {
const seq = this.nextSequence(depthMessage.s);
const message = {
type: 'level',
productId: genericProduct,
time: new Date(+depthMessage.E),
price: level[0],
size: level[1],
sequence: seq,
side: 'buy',
count: 1
};
this.push(message);
});
depthMessage.a.forEach((level) => {
const seq = this.nextSequence(depthMessage.s);
const message = {
type: 'level',
productId: genericProduct,
time: new Date(+depthMessage.E),
price: level[0],
size: level[1],
sequence: seq,
side: 'sell',
count: 1
};
this.push(message);
});
}
onOpen() {
// Do nothing for now
}
createSnapshotMessage(msg) {
this.sequences[msg.s] = 0;
const orders = {};
const snapshotMessage = {
type: 'snapshot',
time: new Date(),
productId: BinanceAPI_1.BinanceAPI.genericProduct(msg.s),
sequence: 0,
sourceSequence: msg.lastUpdateId,
asks: [],
bids: [],
orderPool: orders
};
msg.bids.forEach((level) => {
let price = level[0];
let size = level[1];
const newOrder = {
id: price,
price: types_1.Big(price),
size: types_1.Big(size),
side: 'buy'
};
const priceLevel = {
price: types_1.Big(price),
totalSize: types_1.Big(size),
orders: [newOrder]
};
snapshotMessage.bids.push(priceLevel);
orders[newOrder.id] = newOrder;
});
msg.asks.forEach((level) => {
let price = level[0];
let size = level[1];
const newOrder = {
id: price,
price: types_1.Big(price),
size: types_1.Big(size),
side: 'sell'
};
const priceLevel = {
price: types_1.Big(price),
totalSize: types_1.Big(size),
orders: [newOrder]
};
snapshotMessage.asks.push(priceLevel);
orders[newOrder.id] = newOrder;
});
return snapshotMessage;
}
}
exports.BinanceFeed = BinanceFeed;