intrinio-realtime
Version:
Intrinio SDK for Real-Time Stock Prices
1,255 lines (1,166 loc) • 45.3 kB
JavaScript
const encoder = new TextEncoder();
const decoder = new TextDecoder("utf8");
const unicodeDecoder = new TextDecoder("utf-16le");
const SELF_HEAL_BACKOFFS = [10000, 30000, 60000, 300000, 600000];
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
const CLIENT_INFO_HEADER_KEY = "Client-Information";
const CLIENT_INFO_HEADER_VALUE = "IntrinioRealtimeNodeSDKv5.4";
const MESSAGE_VERSION_HEADER_KEY = "UseNewEquitiesFormat";
const MESSAGE_VERSION_HEADER_VALUE = "v2";
const EVENT_BUFFER_SIZE = 100;
async function doBackoff(context, callback) {
let i = 0;
let backoff = SELF_HEAL_BACKOFFS[i];
let success = true;
await callback.call(context).catch(() => success = false);
while (!success) {
console.log("Intrinio Realtime Client - Sleeping for %isec", (backoff/1000));
await sleep(backoff);
i = Math.min(i + 1, SELF_HEAL_BACKOFFS.length - 1);
backoff = SELF_HEAL_BACKOFFS[i];
success = true;
await callback.call(context).catch(() => success = false);
}
}
function readString(bytes, startPos, endPos) {
if (startPos < 0) startPos = 0;
else if (startPos >= bytes.length) return '';
else startPos |= 0;
if (endPos === undefined || endPos > bytes.length) endPos = bytes.length;
else endPos |= 0;
if (endPos <= startPos) return '';
const chunk = bytes.slice(startPos, endPos);
return decoder.decode(chunk);
}
function readUnicodeString(bytes, startPos, endPos) {
if (startPos < 0) startPos = 0;
else if (startPos >= bytes.length) return '';
else startPos |= 0;
if (endPos === undefined || endPos > bytes.length) endPos = bytes.length;
else endPos |= 0;
if (endPos <= startPos) return '';
const chunk = bytes.slice(startPos, endPos);
return unicodeDecoder.decode(chunk);
}
function readInt32(bytes, startPos = 0) {
const first = bytes[startPos];
const last = bytes[startPos + 3];
if (first === undefined || last === undefined)
console.error("Intrinio Realtime Client - Cannot read UInt32");
return (
first +
(bytes[++startPos] * 256) +
(bytes[++startPos] * 65536) +
(last << 24)
);
}
function readUInt32(bytes, startPos = 0) {
const first = bytes[startPos];
const last = bytes[startPos + 3];
if (first === undefined || last === undefined)
console.error("Intrinio Realtime Client - Cannot read UInt32");
return (
first +
(bytes[++startPos] * 256) +
(bytes[++startPos] * 65536) +
(last * 16777216)
);
}
function readFloat32(bytes, float32Array, backingByteArray, startPos = 0) {
const first = bytes[startPos];
const last = bytes[startPos + 3];
if (first === undefined || last === undefined)
console.error("Intrinio Realtime Client - Cannot read Float32");
backingByteArray[0] = first;
backingByteArray[1] = bytes[++startPos];
backingByteArray[2] = bytes[++startPos];
backingByteArray[3] = last;
let parsed = parseFloat(float32Array[0].toFixed(4));
return parsed < 0 ? 0 : parsed;
}
function readUInt64(bytes, startPos = 0) {
const first = bytes[startPos];
const last = bytes[startPos + 7];
if (first === undefined || last === undefined)
console.error("Intrinio Realtime Client - Cannot read UInt64");
const lower =
first +
(bytes[++startPos] * 256) +
(bytes[++startPos] * 65536) +
(bytes[++startPos] * 16777216);
const upper =
bytes[++startPos] +
(bytes[++startPos] * 256) +
(bytes[++startPos] * 65536) +
(last * 16777216);
return (BigInt(lower) + (BigInt(upper) << 32n));
}
function writeString(bytes, string, startPos) {
if (startPos === undefined || startPos < 0 || startPos > bytes.length - 1) return bytes;
const encodedString = encoder.encode(string);
const bytesAvailable = bytes.length - startPos;
if (bytesAvailable < string.length) {
const trimmedEncodedString = encodedString.slice(0, bytesAvailable);
bytes.set(trimmedEncodedString, startPos);
}
else bytes.set(encodedString, startPos);
}
function copyInto(source, destination, destinationStartIndex){
for (let i = 0; i < source.length; i++){
destination[destinationStartIndex + i] = source[i]
}
}
async function readFileChunk(fileStream, chunkSize){
const buffer = Buffer.alloc(chunkSize);
const buff = await fileStream.read(buffer, 0, chunkSize, null);
if (buff === undefined || buff === null || buff.bytesRead === 0 || buff.buffer === undefined || buff.buffer === null){
return null;
}
else
return buff.buffer;
}
async function * replayTickFileWithoutDelay(filePath){
if (require('fs').existsSync(filePath)) {
const fileStream = await require('fs').promises.open(filePath, 'r');
let readResult = await readFileChunk(fileStream, 1);
while (readResult !== undefined && readResult !== null){
let eventBytes= Buffer.alloc(EVENT_BUFFER_SIZE);
eventBytes[0] = 1; // This is the number of messages in the group
eventBytes[1] = readResult[0]; // This is message type
eventBytes[2] = (await readFileChunk(fileStream, 1))[0]; // This is message length, including this and the previous byte.
copyInto(await readFileChunk(fileStream, eventBytes[2] - 2), eventBytes, 3); // read the rest of the message
let timeReceivedBytes = await readFileChunk(fileStream, 8);
let timeReceived = readUInt64(timeReceivedBytes, 0);
yield new Tick(timeReceived, eventBytes);
readResult = await readFileChunk(fileStream, 1);
}
fileStream.close();
}
else
yield null;
}
async function fillNextTicks(enumerators, nextTicks){
for (let i = 0; i < nextTicks.length; i++){
if (nextTicks[i] === undefined || nextTicks[i] === null){
let next = await enumerators[i].next();
if (!next.done) {
nextTicks[i] = next.value;
}
else{
nextTicks[i] = null;
}
}
}
}
function pullNextTick(nextTicks){
let pullIndex = 0;
let t = 9223372036854775806; //max value
for (let i = 0; i < nextTicks.length; i++){
if (nextTicks[i] !== undefined && nextTicks[i] !== null && nextTicks[i].timeReceived < t){
pullIndex = i;
t = nextTicks[i].timeReceived;
}
}
let pulledTick = nextTicks[pullIndex];
nextTicks[pullIndex] = null;
return pulledTick;
}
function hasAnyValue(nextTicks){
let hasValue = false;
for (let i = 0; i < nextTicks.length; i++){
if (nextTicks[i] !== undefined && nextTicks[i] !== null){
hasValue = true;
}
}
return hasValue;
}
async function * replayFileGroupWithoutDelay(tickGroup) {
let nextTicks = Array(tickGroup.length);
for(let i = 0; i < tickGroup.length; i++){
nextTicks[i] = null;
}
let enumerators = Array(tickGroup.length);
for(let i = 0; i < tickGroup.length; i++){
nextTicks[i] = null;
}
for (let i = 0; i < tickGroup.length; i++) {
enumerators[i] = tickGroup[i]
}
await fillNextTicks(enumerators, nextTicks)
while (hasAnyValue(nextTicks)){
let nextTick = pullNextTick(nextTicks);
if (nextTick !== undefined && nextTick !== null){
yield nextTick;
}
await fillNextTicks(enumerators, nextTicks);
}
}
async function * replayFileGroupWithDelay(allTicks){
const multiplier = BigInt(1000000);
const start = BigInt(new Date().getTime()) * multiplier; //getTime returns milliseconds since epoch
let offset = BigInt(0);
let aggregatedTickIterator = await replayFileGroupWithoutDelay(allTicks);
let next = await aggregatedTickIterator.next();
if (offset === BigInt(0)) {
offset = start - next.value.timeReceived;
}
yield next.value;
while(!next.done){
next = await aggregatedTickIterator.next();
// sleep until the tick happens
const now = BigInt(new Date().getTime()) * multiplier;
if ((next.value.timeReceived + offset) <= now) {
let sleepTime = (now - (next.value.timeReceived + offset)) / multiplier
await new Promise(r => setTimeout(r, Number(sleepTime))); //input is milliseconds
}
yield next.value
}
}
function writeCsvHeaderRow(csvWriter){
csvWriter.write("\"Type\",\"Symbol\",\"Price\",\"Size\",\"Timestamp\",\"TotalVolume\",\"SubProvider\",\"MarketCenter\",\"Condition\"\r\n");
}
function writeCsvTradeRow(csvWriter, trade){
csvWriter.write(`\"${trade.Type}\",\"${trade.Symbol}\",\"${trade.Price}\",\"${trade.Size}\",\"${trade.Timestamp.toString()}\",\"${trade.TotalVolume}\",\"${trade.SubProvider}\",\"${trade.MarketCenter}\",\"${trade.Condition}\"\r\n`);
}
function writeCsvQuoteRow(csvWriter, quote){
csvWriter.write(`\"${quote.Type}\",\"${quote.Symbol}\",\"${quote.Price}\",\"${quote.Size}\",\"${quote.Timestamp.toString()}\",\"\",\"${quote.SubProvider}\",\"${quote.MarketCenter}\",\"${quote.Condition}\"\r\n`);
}
async function replayToCsv(outputFilePath, config, channels, tradesOnly, apiKey){
console.log("Creating file " + outputFilePath);
let csvWriter = require('fs').createWriteStream(outputFilePath, {flags: 'a'});
writeCsvHeaderRow(csvWriter);
function onTrade(trade){
writeCsvTradeRow(csvWriter, trade);
}
function onQuote(quote){
writeCsvQuoteRow(csvWriter, quote);
}
let client = await new this.ReplayClient(apiKey, onTrade, onQuote, config);
await client.join(channels, tradesOnly);
}
const defaultConfig = {
provider: 'IEX', //IEX (same as) REALTIME, or DELAYED_SIP, or NASDAQ_BASIC, or CBOE_ONE, or MANUAL
ipAddress: undefined,
tradesOnly: false,
isPublicKey: false,
delayed: false //set to true if you have realtime access and want to force delayed mode. Otherwise, if you only have delayed, you'll get delayed not matter if you set this or not.
};
const defaultReplayConfig = {
provider: 'IEX', //IEX (same as) REALTIME, or DELAYED_SIP, or NASDAQ_BASIC, or CBOE_ONE, or MANUAL
ipAddress: undefined,
tradesOnly: false,
isPublicKey: false,
replayDate: '2025-01-08',
replayAsIfLive: false,
replayDeleteFileWhenDone: true
};
class IntrinioRealtime {
constructor(accessKey, onTrade, onQuote, config = {}) {
this._accessKey = accessKey;
this._config = Object.assign(defaultConfig, config);
this._token = null;
this._websocket = null;
this._isReady = false;
this._attemptingReconnect = false;
this._lastReset = Date.now();
this._msgCount = 0;
this._channels = new Map();
this._onTrade = (onTrade && (typeof onTrade === "function")) ? onTrade : (_) => {};
this._onQuote = (onQuote && (typeof onQuote === "function")) ? onQuote : (_) => {};
this._float32Array = new Float32Array(1);
this._backingByteArray = new Uint8Array(this._float32Array.buffer);
if ((!this._accessKey) || (this._accessKey === "")) {
throw "Intrinio Realtime Client - Access Key is required";
}
if (!this._config.provider) {
throw "Intrinio Realtime Client - 'config.provider' must be specified";
}
else if ((this._config.provider !== "IEX") && (this._config.provider !== "CBOE_ONE") && (this._config.provider !== "REALTIME") && (this._config.provider !== "MANUAL")
&& (this._config.provider !== "DELAYED_SIP") && (this._config.provider !== "NASDAQ_BASIC")) {
throw "Intrinio Realtime Client - 'config.provider' must be either 'IEX' or 'REALTIME' or 'CBOE_ONE' or 'MANUAL' or 'DELAYED_SIP' or 'NASDAQ_BASIC'";
}
if ((this._config.provider === "MANUAL") && ((!this._config.ipAddress) || (this._config.ipAddress === ""))) {
throw "Intrinio Realtime Client - 'config.ipAddress' must be specified for manual configuration";
}
if(!onTrade) {
throw "Intrinio Realtime Client - 'onTrade' callback is required";
}
if(!onQuote){
this._config.tradesOnly = true;
}
doBackoff(this, this._trySetToken).then(
() => {doBackoff(this, this._resetWebsocket).then(
() => {
console.log("Intrinio Realtime Client - Startup succeeded");
if (!this._config.isPublicKey) {
process.on('SIGINT', () => {
console.log("Intrinio Realtime Client - Shutdown detected");
this.stop();
process.kill(process.pid, 'SIGTERM');})
}
},
() => {console.error("Intrinio Realtime Client - Startup failed. Unable to establish websocket connection.");})},
() => {console.error("Intrinio Realtime Client - Startup failed. Unable to acquire auth token.");});
}
_getAuthUrl() {
switch(this._config.provider) {
case "REALTIME":
if (this._config.isPublicKey) return "https://realtime-mx.intrinio.com/auth";
else return "https://realtime-mx.intrinio.com/auth?api_key=" + this._accessKey;
case "IEX":
if (this._config.isPublicKey) return "https://realtime-mx.intrinio.com/auth";
else return "https://realtime-mx.intrinio.com/auth?api_key=" + this._accessKey;
case "DELAYED_SIP":
if (this._config.isPublicKey) return "https://realtime-delayed-sip.intrinio.com/auth";
else return "https://realtime-delayed-sip.intrinio.com/auth?api_key=" + this._accessKey;
case "NASDAQ_BASIC":
if (this._config.isPublicKey) return "https://realtime-nasdaq-basic.intrinio.com/auth";
else return "https://realtime-nasdaq-basic.intrinio.com/auth?api_key=" + this._accessKey;
case "CBOE_ONE":
if (this._config.isPublicKey) return "https://cboe-one.intrinio.com/auth";
else return "https://cboe-one.intrinio.com/auth?api_key=" + this._accessKey;
case "MANUAL":
if (this._config.isPublicKey) return "http://" + this._config.ipAddress + "/auth";
else return "http://" + this._config.ipAddress + "/auth?api_key=" + this._accessKey;
default: throw "Intrinio Realtime Client - 'config.provider' not specified!";
}
}
_getWebSocketUrl() {
let delayed_part = this._config.delayed ? "&delayed=true" : "";
switch(this._config.provider) {
case "REALTIME": return `wss://realtime-mx.intrinio.com/socket/websocket?vsn=1.0.0&token=${this._token}&${CLIENT_INFO_HEADER_KEY}=${CLIENT_INFO_HEADER_VALUE}&${MESSAGE_VERSION_HEADER_KEY}=${MESSAGE_VERSION_HEADER_VALUE}${delayed_part}`;
case "IEX": return `wss://realtime-mx.intrinio.com/socket/websocket?vsn=1.0.0&token=${this._token}&${CLIENT_INFO_HEADER_KEY}=${CLIENT_INFO_HEADER_VALUE}&${MESSAGE_VERSION_HEADER_KEY}=${MESSAGE_VERSION_HEADER_VALUE}${delayed_part}`;
case "DELAYED_SIP": return `wss://realtime-delayed-sip.intrinio.com/socket/websocket?vsn=1.0.0&token=${this._token}&${CLIENT_INFO_HEADER_KEY}=${CLIENT_INFO_HEADER_VALUE}&${MESSAGE_VERSION_HEADER_KEY}=${MESSAGE_VERSION_HEADER_VALUE}${delayed_part}`;
case "NASDAQ_BASIC": return `wss://realtime-nasdaq-basic.intrinio.com/socket/websocket?vsn=1.0.0&token=${this._token}&${CLIENT_INFO_HEADER_KEY}=${CLIENT_INFO_HEADER_VALUE}&${MESSAGE_VERSION_HEADER_KEY}=${MESSAGE_VERSION_HEADER_VALUE}${delayed_part}`;
case "CBOE_ONE": return `wss://cboe-one.intrinio.com/socket/websocket?vsn=1.0.0&token=${this._token}&${CLIENT_INFO_HEADER_KEY}=${CLIENT_INFO_HEADER_VALUE}&${MESSAGE_VERSION_HEADER_KEY}=${MESSAGE_VERSION_HEADER_VALUE}${delayed_part}`;
case "MANUAL": return "ws://" + this._config.ipAddress + `/socket/websocket?vsn=1.0.0&token=${this._token}&${CLIENT_INFO_HEADER_KEY}=${CLIENT_INFO_HEADER_VALUE}&${MESSAGE_VERSION_HEADER_KEY}=${MESSAGE_VERSION_HEADER_VALUE}${delayed_part}`;
default: throw "Intrinio Realtime Client - 'config.provider' not specified!";
}
}
_getMessageType(val){
switch(val) {
case 0:
return 'Trade';
break;
case 1:
return 'Ask';
break;
case 2:
return 'Bid';
break;
default:
return '';
break;
}
}
_getSubProvider(val){
switch(val) {
case 0:
return 'NONE';
break;
case 1:
return 'CTA_A';
break;
case 2:
return 'CTA_B';
break;
case 3:
return 'UTP';
break;
case 4:
return 'OTC';
break;
case 5:
return 'NASDAQ_BASIC';
break;
case 6:
return 'IEX';
break;
case 7:
return 'CBOE_ONE';
break;
default:
return 'NONE';
break;
}
}
_parseTrade(bytes) {
let symbolLength = bytes[2];
let conditionLength = bytes[26 + symbolLength];
let marketCenter = readUnicodeString(bytes, 4 + symbolLength, 6 + symbolLength);
let subProvider = this._getSubProvider(bytes[3 + symbolLength]);
let isDarkpool = false;
switch(marketCenter) {
case 'CTA_A':
case 'CTA_B':
case 'UTP':
case 'OTC':
isDarkpool = (!marketCenter) || marketCenter == "\0" || marketCenter == "D" || marketCenter == "E" || marketCenter.trim().length === 0;
break;
case 'NASDAQ_BASIC':
isDarkpool = (!marketCenter) || marketCenter == "\0" || marketCenter == "L" || marketCenter == "2" || marketCenter.trim().length === 0;
break;
default:
isDarkpool = false;
break;
}
return {
Type: this._getMessageType(bytes[0]),
Symbol: readString(bytes, 3, 3 + symbolLength),
Price: readFloat32(bytes, this._float32Array, this._backingByteArray, 6 + symbolLength),
Size: readUInt32(bytes, 10 + symbolLength),
Timestamp: readUInt64(bytes, 14 + symbolLength),
TotalVolume: readUInt32(bytes, 22 + symbolLength),
SubProvider: subProvider,
MarketCenter: marketCenter,
Condition: conditionLength > 0 ? readString(bytes, 27 + symbolLength, 27 + symbolLength + conditionLength) : "",
IsDarkpool: isDarkpool
}
}
_parseQuote (bytes) {
let symbolLength = bytes[2];
let conditionLength = bytes[22 + symbolLength];
let marketCenter = readUnicodeString(bytes, 4 + symbolLength, 6 + symbolLength);
return {
Type: this._getMessageType(bytes[0]),
Symbol: readString(bytes, 3, 3 + symbolLength),
Price: readFloat32(bytes, this._float32Array, this._backingByteArray, 6 + symbolLength),
Size: readUInt32(bytes, 10 + symbolLength),
Timestamp: readUInt64(bytes, 14 + symbolLength),
SubProvider: this._getSubProvider(bytes[3 + symbolLength]),
MarketCenter: marketCenter,
Condition: conditionLength > 0 ? readString(bytes, 23 + symbolLength, 23 + symbolLength + conditionLength) : ""
}
}
_parseSocketMessage(data) {
let bytes = new Uint8Array(data);
let msgCount = bytes[0];
let startIndex = 1;
for (let i = 0; i < msgCount; i++) {
let msgType = bytes[startIndex]
let msgLength = bytes[startIndex + 1]
let endIndex = startIndex + msgLength
let chunk = bytes.slice(startIndex, endIndex)
switch(msgType) {
case 0:
let trade = this._parseTrade(chunk)
this._onTrade(trade)
break;
case 1:
case 2:
let quote = this._parseQuote(chunk)
this._onQuote(quote)
break;
default: console.warn("Intrinio Realtime Client - Invalid message type: %i", msgType)
}
startIndex = endIndex
}
}
_trySetToken() {
if (this._config.isPublicKey)
return new Promise((fulfill, reject) => {
try {
console.log("Intrinio Realtime Client - Authorizing (public key)...");
const url = this._getAuthUrl();
const xhr = new XMLHttpRequest();
//TODO - add header
xhr.onerror = (error) => {
console.error("Intrinio Realtime Client - Error getting public key auth token: ", error);
reject();
};
xhr.ontimeout = () => {
console.error("Intrinio Realtime Client - Timed out trying to get auth token.");
reject();
};
xhr.onabort = () => {
console.error("Intrinio Realtime Client - Aborted attempt to get auth token.");
reject();
};
xhr.onreadystatechange = () => {
if (xhr.readyState === 4) {
if (xhr.status === 401) {
console.error("Intrinio Realtime Client - Unable to authorize (public key)");
reject();
}
else if (xhr.status !== 200) {
console.error("Intrinio Realtime Client - Could not get public key auth token: Status code (%i)", xhr.status);
reject();
}
else {
console.log("Intrinio Realtime Client - Authorized (public key)");
this._token = xhr.responseText;
fulfill();
}
}
}
xhr.open("GET", url, true);
xhr.overrideMimeType("text/html");
xhr.setRequestHeader('Content-Type', 'application/json');
xhr.setRequestHeader('Authorization', 'Public ' + this._accessKey);
xhr.setRequestHeader(CLIENT_INFO_HEADER_KEY, CLIENT_INFO_HEADER_VALUE);
xhr.send();
}
catch (error) {
console.error("Intrinio Realtime Client - Error in authorization (%s)", error);
reject();
}
})
else
return new Promise((fulfill, reject) => {
try {
const protocol = (this._config.provider === "MANUAL") ? require('http') : require('https');
console.log("Intrinio Realtime Client - Authorizing...");
const url = this._getAuthUrl();
const options = {
headers: {
[CLIENT_INFO_HEADER_KEY]: CLIENT_INFO_HEADER_VALUE
}
};
const request = protocol.get(url, options, response => {
if (response.statusCode == 401) {
console.error("Intrinio Realtime Client - Unable to authorize");
reject();
}
else if (response.statusCode != 200) {
console.error("Intrinio Realtime Client - Could not get auth token: Status code (%i)", response.statusCode);
reject();
}
else {
response.on("data", data => {
this._token = decoder.decode(data);
console.log("Intrinio Realtime Client - Authorized");
fulfill();
});
}
})
request.on("timeout", () => {
console.error("Intrinio Realtime Client - Timed out trying to get auth token.");
reject();
});
request.on("error", error => {
console.error("Intrinio Realtime Client - Error getting auth token: ", error);
reject();
});
}
catch (error) {
console.error("Intrinio Realtime Client - Error in authorization (%s)", error);
reject();
}
});
}
_makeJoinMessage(tradesOnly, symbol) {
let message = null;
switch (symbol) {
case "$lobby":
message = new Uint8Array(11);
message[0] = 74;
if (tradesOnly) {
message[1] = 1;
} else {
message[1] = 0;
}
writeString(message, "$FIREHOSE", 2);
return message;
default:
message = new Uint8Array(2 + symbol.length);
message[0] = 74;
if (tradesOnly) {
message[1] = 1;
} else {
message[1] = 0;
}
writeString(message, symbol, 2);
return message;
}
}
_makeLeaveMessage(symbol) {
let message = null;
switch (symbol) {
case "$lobby":
message = new Uint8Array(10);
message[0] = 76;
writeString(message, "$FIREHOSE", 1);
return message;
default:
message = new Uint8Array(1 + symbol.length);
message[0] = 76;
writeString(message, symbol, 1);
return message;
}
}
_resetWebsocket() {
if (this._config.isPublicKey)
return new Promise((fulfill, reject) => {
try {
console.info("Intrinio Realtime Client - Websocket initializing (public key)");
let wsUrl = this._getWebSocketUrl();
this._websocket = new WebSocket(wsUrl);
this._websocket.binaryType = "arraybuffer";
this._websocket.onopen = () => {
console.log("Intrinio Realtime Client - Websocket connected (public key)");
if (this._channels.size > 0) {
for (const [channel, tradesOnly] of this._channels) {
let message = this._makeJoinMessage(tradesOnly, channel);
console.info("Intrinio Realtime Client - Joining channel: %s (trades only = %s)", channel, tradesOnly);
this._websocket.send(message);
}
}
this._lastReset = Date.now();
this._isReady = true;
this._isReconnecting = false;
fulfill(true);
};
this._websocket.onclose = (code) => {
if (!this._attemptingReconnect) {
this._isReady = false;
console.info("Intrinio Realtime Client - Websocket closed (code: %o)", code);
if (code != 1000) {
console.info("Intrinio Realtime Client - Websocket reconnecting...");
if (!this._isReady) {
this._attemptingReconnect = true;
if ((Date.now() - this._lastReset) > 86400000) {
doBackoff(this, this._trySetToken).then(() => {doBackoff(this, this._resetWebsocket)});
}
doBackoff(this, this._resetWebsocket);
}
}
}
else reject();
}
this._websocket.onerror = (error) => {
console.error("Intrinio Realtime Client - Websocket error: %s", error);
reject();
}
this._websocket.onmessage = (message) => {
this._msgCount++;
if (message.data instanceof ArrayBuffer)
this._parseSocketMessage(message.data);
else
console.log("Intrinio Realtime Client - Message: %s", message.data);
};
}
catch (error) {
console.error("Intrinio Realtime Client - Error establishing public key websocket connection (%s)", error);
reject();
}
});
else
return new Promise((fulfill, reject) => {
try {
console.info("Intrinio Realtime Client - Websocket initializing");
const WebSocket = require('ws');
let wsUrl = this._getWebSocketUrl();
this._websocket = new WebSocket(wsUrl, {perMessageDeflate: false}, {headers: {
[CLIENT_INFO_HEADER_KEY]: CLIENT_INFO_HEADER_VALUE,
[MESSAGE_VERSION_HEADER_KEY]: MESSAGE_VERSION_HEADER_VALUE
}});
this._websocket.binaryType = "arraybuffer";
this._websocket.on("open", () => {
console.log("Intrinio Realtime Client - Websocket connected");
if (this._channels.size > 0) {
for (const [channel, tradesOnly] of this._channels) {
let message = this._makeJoinMessage(tradesOnly, channel);
console.info("Intrinio Realtime Client - Joining channel: %s (trades only = %s)", channel, tradesOnly);
this._websocket.send(message);
}
}
this._lastReset = Date.now();
this._isReady = true;
this._isReconnecting = false;
fulfill(true);
});
this._websocket.on("close", (code, reason) => {
if (!this._attemptingReconnect) {
this._isReady = false;
console.info("Intrinio Realtime Client - Websocket closed (code: %o)", code);
if (code != 1000) {
console.info("Intrinio Realtime Client - Websocket reconnecting...");
if (!this._isReady) {
this._attemptingReconnect = true;
if ((Date.now() - this._lastReset) > 86400000) {
doBackoff(this, this._trySetToken).then(() => {doBackoff(this, this._resetWebsocket)});
}
doBackoff(this, this._resetWebsocket);
}
}
}
else reject();
});
this._websocket.on("error", (error) => {
console.error("Intrinio Realtime Client - Websocket error: %s", error);
reject();
});
this._websocket.on("message", (message) => {
this._msgCount++;
this._parseSocketMessage(message);
});
}
catch (error) {
console.error("Intrinio Realtime Client - Error establishing websocket connection (%s)", error);
reject();
}
});
}
_join(symbol, tradesOnly) {
if (this._channels.has("$lobby")) {
console.warn("Intrinio Realtime Client - $lobby channel already joined. Other channels not necessary.");
}
if (!this._channels.has(symbol)) {
this._channels.set(symbol, tradesOnly);
let message = this._makeJoinMessage(tradesOnly, symbol);
console.info("Intrinio Realtime Client - Joining channel: %s (trades only = %s)", symbol, tradesOnly);
this._websocket.send(message);
}
}
_leave(symbol) {
if (this._channels.has(symbol)) {
this._channels.delete(symbol);
let message = this._makeLeaveMessage(symbol);
console.info("Intrinio Realtime Client - Leaving channel: %s", symbol);
this._websocket.send(message);
}
}
async join(symbols, tradesOnly) {
while (!this._isReady) {
await sleep(1000);
}
let _tradesOnly = (this._config.tradesOnly ? this._config.tradesOnly : false) || (tradesOnly ? tradesOnly : false);
if (symbols instanceof Array) {
for (const symbol of symbols) {
if (!this._channels.has(symbol)){
this._join(symbol, _tradesOnly);
}
}
}
else if (typeof symbols === "string") {
if (!this._channels.has(symbols)){
this._join(symbols, _tradesOnly);
}
}
else if ((typeof tradesOnly !== "undefined") || (typeof tradesOnly !== "boolean")) {
console.error("Intrinio Realtime Client - If provided, 'tradesOnly' must be of type 'boolean', not '%s'", typeof tradesOnly);
}
else {
console.error("Intrinio Realtime Client - Invalid use of 'join'");
}
}
leave(symbols) {
if (symbols instanceof Array) {
for (const symbol of symbols) {
if (this._channels.has(symbol)) {
this._leave(symbol)
}
}
}
else if (typeof symbols === "string") {
if (this._channels.has(symbols)) {
this._leave(symbols);
}
}
else {
if (arguments.length == 0) {
for (const channel of this._channels.keys()) {
this._leave(channel);
}
}
else {
console.error("Intrinio Realtime Client - Invalid use of 'leave'");
}
}
}
async stop() {
this._isReady = false;
console.log("Intrinio Realtime Client - Leaving subscribed channels");
for (const channel of this._channels.keys()) {
this._leave(channel);
}
while (this._websocket.bufferedAmount > 0) {
await sleep(500);
}
console.log("Intrinio Realtime Client - Websocket closing");
if (this._websocket) {
this._websocket.close(1000, "Terminated by client");
}
}
getTotalMsgCount() {
return this._msgCount;
}
}
class Tick{
constructor(timeReceived, data){
this.timeReceived = timeReceived;
this.data = data;
}
}
class IntrinioRealtimeReplayClient {
constructor(accessKey, onTrade, onQuote, config = {}) {
this._accessKey = accessKey;
this._config = Object.assign(defaultReplayConfig, config);
this._msgCount = 0;
this._channels = new Map();
this._onTrade = (onTrade && (typeof onTrade === "function")) ? onTrade : (_) => {};
this._onQuote = (onQuote && (typeof onQuote === "function")) ? onQuote : (_) => {};
this._float32Array = new Float32Array(1);
this._backingByteArray = new Uint8Array(this._float32Array.buffer);
if (this._config.isPublicKey){
throw "Intrinio Replay Client - Replay Client is only available in a node.js environment.";
}
if ((!this._accessKey) || (this._accessKey === "")) {
throw "Intrinio Replay Client - Access Key is required";
}
if (!this._config.provider) {
throw "Intrinio Replay Client - 'config.provider' must be specified";
}
else if ((this._config.provider !== "IEX") && (this._config.provider !== "CBOE_ONE") && (this._config.provider !== "REALTIME") && (this._config.provider !== "MANUAL")
&& (this._config.provider !== "DELAYED_SIP") && (this._config.provider !== "NASDAQ_BASIC")) {
throw "Intrinio Replay Client - 'config.provider' must be either 'IEX' or 'REALTIME' or 'CBOE_ONE' or 'MANUAL' or 'DELAYED_SIP' or 'NASDAQ_BASIC'";
}
if(!onTrade) {
throw "Intrinio Replay Client - 'onTrade' callback is required";
}
if(!onQuote){
this._config.tradesOnly = true;
}
this._start();
}
async _getAllApiDownloadResponses(urls){
let responses = [];
for (let i = 0; i < urls.length; i++){
try{
let apiResponse = await this._getApiDownloadResponse(urls[i]);
if (apiResponse !== undefined && apiResponse !== null){ responses.push(apiResponse); }
}
catch (ex){
//not all customers have access to all subsources within a source. Ignore errors if they don't.
}
}
return responses;
}
async _getAllFilePaths(responses){
let filePaths = [];
//download the files to a temp dir
for (let i = 0; i < responses.length; i++){
let filePath = await this._downloadFile(this._decodeUrl(responses[i].url), responses[i].name)
if (filePath !== undefined && filePath !== null && filePath !== ""){ filePaths.push(filePath); }
}
return filePaths;
}
_deleteReplayFiles(filePaths){
const fs = require('fs');
for (let i = 0; i < filePaths.length; i++) {
if (fs.existsSync(filePaths[i])){
console.log("Intrinio Replay Client - Deleting file: " + filePath);
fs.unlinkSync(filePaths[i]); //delete file
}
}
}
async _getAllFileIterators(filePaths){
let enumerators = [];
for (let i = 0; i < filePaths.length; i++){
enumerators.push(await replayTickFileWithoutDelay(filePaths[i]));
}
return enumerators;
}
async _getAggregateTickIterator(fileGroup){
if (this._config.replayAsIfLive){
return await replayFileGroupWithDelay(fileGroup);
}
else{
return await replayFileGroupWithoutDelay(fileGroup);
}
}
async _start(){
let urls = await this._getApiReplayUrls();
let responses = await this._getAllApiDownloadResponses(urls);
let filePaths = await this._getAllFilePaths(responses);
let fileGroup = await this._getAllFileIterators(filePaths);
let aggregatedTickIterator = await this._getAggregateTickIterator(fileGroup);
console.log("Starting to read files...")
let next = await aggregatedTickIterator.next();
await this._parseSocketMessage(next.value.data);
while(!next.done){
next = await aggregatedTickIterator.next();
await this._parseSocketMessage(next.value.data);
}
if (this._config.replayDeleteFileWhenDone){
console.log("cleaning up files");
this._deleteReplayFiles(filePaths, fs)
}
}
_decodeUrl(url){
if (url !== undefined){
return url.replace('\u0026', '&');
}
return url;
}
async _downloadFile(url, fileName){
const os = require('os');
const fs = require('fs');
const https = require('https');
const path = require('path');
let tempDir = os.tmpdir();
let filePath = path.join(tempDir, fileName);
console.log("Intrinio Replay Client - Starting to download to " + filePath + ".");
return new Promise((fulfill, reject) => {
try {
const request = https.get(url, {}, response => {
if (response.statusCode != 200) {
console.error("Intrinio Replay Client - Could not download " + fileName + ": Status code (%i)", response.statusCode);
reject();
}
else {
const fileStream = fs.createWriteStream(filePath);
response.pipe(fileStream);
fileStream.on("finish", () => {
fileStream.close();
console.log("Intrinio Replay Client - Successfully downloaded " + filePath + ".");
fulfill(filePath);
});
}
})
request.on("timeout", () => {
console.error("Intrinio Replay Client - Timed out trying to download " + fileName + ".");
reject();
});
request.on("error", error => {
console.error("Intrinio Replay Client - Error downloading " + fileName + ": (%s)", error);
reject();
});
}
catch (error) {
console.error("Intrinio Replay Client - Error downloading " + fileName + ": (%s)", error);
reject();
}
});
}
async _getApiDownloadResponse(url) {
return new Promise((fulfill, reject) => {
try {
const protocol = require('https');
const request = protocol.get(url, {}, response => {
if (response.statusCode != 200) {
//console.error("Intrinio Replay Client - Could not fetch download URL from API: Status code (%i)", response.statusCode);
reject();
}
else {
response.on("data", data => {
let apiResponse = JSON.parse(data);
console.log("Intrinio Replay Client - Successfully fetched download URL from API");
fulfill(apiResponse);
});
}
})
request.on("timeout", () => {
//console.error("Intrinio Replay Client - Timed out trying to fetched download URL from API.");
reject();
});
request.on("error", error => {
console.error("Intrinio Replay Client - Error fetching download URL from API: ", error);
reject();
});
}
catch (error) {
console.error("Intrinio Replay Client - Error in fetching download URL (%s)", error);
reject();
}
});
}
_getApiReplayUrls() {
switch(this._config.provider) {
case "REALTIME":
return ["https://api-v2.intrinio.com/securities/replay?subsource=iex&date=" + this._config.replayDate + "&api_key=" + this._accessKey];
case "IEX":
return ["https://api-v2.intrinio.com/securities/replay?subsource=iex&date=" + this._config.replayDate + "&api_key=" + this._accessKey];
case "DELAYED_SIP":
return ["https://api-v2.intrinio.com/securities/replay?subsource=utp_delayed&date=" + this._config.replayDate + "&api_key=" + this._accessKey,
"https://api-v2.intrinio.com/securities/replay?subsource=cta_a_delayed&date=" + this._config.replayDate + "&api_key=" + this._accessKey,
"https://api-v2.intrinio.com/securities/replay?subsource=cta_b_delayed&date=" + this._config.replayDate + "&api_key=" + this._accessKey,
"https://api-v2.intrinio.com/securities/replay?subsource=otc_delayed&date=" + this._config.replayDate + "&api_key=" + this._accessKey,
];
case "NASDAQ_BASIC":
return ["https://api-v2.intrinio.com/securities/replay?subsource=nasdaq_basic&date=" + this._config.replayDate + "&api_key=" + this._accessKey];
case "CBOE_ONE":
return ["https://api-v2.intrinio.com/securities/replay?subsource=cboe_one&date=" + this._config.replayDate + "&api_key=" + this._accessKey];
default: throw "Intrinio Replay Client - 'config.provider' not specified!";
}
}
_getMessageType(val){
switch(val) {
case 0:
return 'Trade';
break;
case 1:
return 'Ask';
break;
case 2:
return 'Bid';
break;
default:
return '';
break;
}
}
_getSubProvider(val){
switch(val) {
case 0:
return 'NONE';
break;
case 1:
return 'CTA_A';
break;
case 2:
return 'CTA_B';
break;
case 3:
return 'UTP';
break;
case 4:
return 'OTC';
break;
case 5:
return 'NASDAQ_BASIC';
break;
case 6:
return 'IEX';
break;
case 7:
return 'CBOE_ONE';
break;
default:
return 'NONE';
break;
}
}
_parseTrade(bytes) {
let symbolLength = bytes[2];
let conditionLength = bytes[26 + symbolLength];
return {
Type: this._getMessageType(bytes[0]),
Symbol: readString(bytes, 3, 3 + symbolLength),
Price: readFloat32(bytes, this._float32Array, this._backingByteArray, 6 + symbolLength),
Size: readUInt32(bytes, 10 + symbolLength),
Timestamp: readUInt64(bytes, 14 + symbolLength),
TotalVolume: readUInt32(bytes, 22 + symbolLength),
SubProvider: this._getSubProvider(bytes[3 + symbolLength]),
MarketCenter: readUnicodeString(bytes, 4 + symbolLength, 6 + symbolLength),
Condition: conditionLength > 0 ? readString(bytes, 27 + symbolLength, 27 + symbolLength + conditionLength) : ""
}
}
_parseQuote (bytes) {
let symbolLength = bytes[2];
let conditionLength = bytes[22 + symbolLength];
return {
Type: this._getMessageType(bytes[0]),
Symbol: readString(bytes, 3, 3 + symbolLength),
Price: readFloat32(bytes, this._float32Array, this._backingByteArray, 6 + symbolLength),
Size: readUInt32(bytes, 10 + symbolLength),
Timestamp: readUInt64(bytes, 14 + symbolLength),
SubProvider: this._getSubProvider(bytes[3 + symbolLength]),
MarketCenter: readUnicodeString(bytes, 4 + symbolLength, 6 + symbolLength),
Condition: conditionLength > 0 ? readString(bytes, 23 + symbolLength, 23 + symbolLength + conditionLength) : ""
}
}
async _parseSocketMessage(data) {
let bytes = new Uint8Array(data);
let msgCount = bytes[0];
let startIndex = 1;
for (let i = 0; i < msgCount; i++) {
let msgType = bytes[startIndex]
let msgLength = bytes[startIndex + 1]
let endIndex = startIndex + msgLength
let chunk = bytes.slice(startIndex, endIndex)
switch(msgType) {
case 0:
let trade = this._parseTrade(chunk);
if (this._channels.has("$lobby") || this._channels.has(trade.Symbol)) {
await this._onTrade(trade);
}
break;
case 1:
case 2:
let quote = this._parseQuote(chunk);
if ((!this._config.tradesOnly) && (this._channels.has("$lobby") || this._channels.has(quote.Symbol))) {
await this._onQuote(quote);
}
break;
default: console.warn("Intrinio Replay Client - Invalid message type: %i", msgType)
}
startIndex = endIndex
}
}
_join(symbol, tradesOnly) {
if (this._channels.has("$lobby")) {
console.warn("Intrinio Replay Client - $lobby channel already joined. Other channels not necessary.");
}
if (!this._channels.has(symbol)) {
this._channels.set(symbol, tradesOnly);
console.info("Intrinio Replay Client - Joining channel: %s (trades only = %s)", symbol, tradesOnly);
}
}
_leave(symbol) {
if (this._channels.has(symbol)) {
this._channels.delete(symbol);
console.info("Intrinio Replay Client - Leaving channel: %s", symbol);
}
}
async join(symbols, tradesOnly) {
let _tradesOnly = (this._config.tradesOnly ? this._config.tradesOnly : false) || (tradesOnly ? tradesOnly : false);
if (symbols instanceof Array) {
for (const symbol of symbols) {
if (!this._channels.has(symbol)){
this._join(symbol, _tradesOnly);
}
}
}
else if (typeof symbols === "string") {
if (!this._channels.has(symbols)){
this._join(symbols, _tradesOnly);
}
}
else if ((typeof tradesOnly !== "undefined") || (typeof tradesOnly !== "boolean")) {
console.error("Intrinio Replay Client - If provided, 'tradesOnly' must be of type 'boolean', not '%s'", typeof tradesOnly);
}
else {
console.error("Intrinio Replay Client - Invalid use of 'join'");
}
}
leave(symbols) {
if (symbols instanceof Array) {
for (const symbol of symbols) {
if (this._channels.has(symbol)) {
this._leave(symbol)
}
}
}
else if (typeof symbols === "string") {
if (this._channels.has(symbols)) {
this._leave(symbols);
}
}
else {
if (arguments.length == 0) {
for (const channel of this._channels.keys()) {
this._leave(channel);
}
}
else {
console.error("Intrinio Replay Client - Invalid use of 'leave'");
}
}
}
async stop() {
console.log("Intrinio Replay Client - Leaving subscribed channels");
for (const channel of this._channels.keys()) {
this._leave(channel);
}
}
getTotalMsgCount() {
return this._msgCount;
}
}
if (typeof window === 'undefined') {
module.exports = { RealtimeClient: IntrinioRealtime, ReplayClient: IntrinioRealtimeReplayClient, replayToCsv: replayToCsv };
}