tardis-machine
Version:
Locally runnable server with built-in data caching, providing both tick-level historical and consolidated real-time cryptocurrency market data via HTTP and WebSocket APIs
110 lines • 5.23 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.streamNormalizedWS = void 0;
const querystring_1 = __importDefault(require("querystring"));
const tardis_dev_1 = require("tardis-dev");
const debug_1 = require("../debug");
const helpers_1 = require("../helpers");
async function streamNormalizedWS(ws, req) {
let messages;
try {
const startTimestamp = new Date().getTime();
const parsedQuery = querystring_1.default.decode(req.getQuery());
const optionsString = parsedQuery['options'];
const streamNormalizedOptions = JSON.parse(optionsString);
(0, debug_1.debug)('WebSocket /ws-stream-normalized started, options: %o', streamNormalizedOptions);
const options = Array.isArray(streamNormalizedOptions) ? streamNormalizedOptions : [streamNormalizedOptions];
let subSequentErrorsCount = {};
let retries = 0;
let bufferedAmount = 0;
const messagesIterables = options.map((option) => {
// let's map from provided options to options and normalizers that needs to be added for dataTypes provided in options
const messages = (0, tardis_dev_1.streamNormalized)({
...option,
withDisconnectMessages: true,
onError: (error) => {
const exchange = option.exchange;
if (subSequentErrorsCount[exchange] === undefined) {
subSequentErrorsCount[exchange] = 0;
}
subSequentErrorsCount[exchange]++;
if (option.withErrorMessages && !ws.closed) {
ws.send(JSON.stringify({
type: 'error',
exchange,
localTimestamp: new Date(),
details: error.message,
subSequentErrorsCount: subSequentErrorsCount[exchange]
}));
}
(0, debug_1.debug)('WebSocket /ws-stream-normalized %s WS connection error: %o', exchange, error);
}
}, ...(0, helpers_1.getNormalizers)(option.dataTypes));
// separately check if any computables are needed for given dataTypes
const computables = (0, helpers_1.getComputables)(option.dataTypes);
if (computables.length > 0) {
return (0, tardis_dev_1.compute)(messages, ...computables);
}
return messages;
});
const filterByDataType = (0, helpers_1.constructDataTypeFilter)(options);
messages = messagesIterables.length === 1 ? messagesIterables[0] : (0, tardis_dev_1.combine)(...messagesIterables);
for await (const message of messages) {
if (ws.closed) {
return;
}
const exchange = message.exchange;
if (subSequentErrorsCount[exchange] !== undefined && subSequentErrorsCount[exchange] >= 50) {
ws.end(1011, `Too many subsequent errors when connecting to ${exchange} WS API`);
return;
}
if (!filterByDataType(message)) {
continue;
}
retries = 0;
bufferedAmount = 0;
// handle backpressure in case of slow clients
while ((bufferedAmount = ws.getBufferedAmount()) > 0) {
retries += 1;
const isState = new Date().valueOf() - message.localTimestamp.valueOf() >= 6;
// log stale messages, stale meaning message was not sent in 6 ms or more (2 retries)
if (isState) {
(0, debug_1.debug)('Slow client, waiting %d ms, buffered amount: %d', 3 * retries, bufferedAmount);
}
if (retries > 300) {
ws.end(1008, 'Too much backpressure');
return;
}
await (0, helpers_1.wait)(3 * retries);
}
ws.send(JSON.stringify(message));
if (message.type !== 'disconnect') {
subSequentErrorsCount[exchange] = 0;
}
}
while (ws.getBufferedAmount() > 0) {
await (0, helpers_1.wait)(100);
}
ws.end(1000, 'WS stream-normalized finished');
const endTimestamp = new Date().getTime();
(0, debug_1.debug)('WebSocket /ws-stream-normalized finished, options: %o, time: %d seconds', streamNormalizedOptions, (endTimestamp - startTimestamp) / 1000);
}
catch (e) {
if (!ws.closed) {
ws.end(1011, e.toString());
}
(0, debug_1.debug)('WebSocket /ws-stream-normalized error: %o', e);
console.error('WebSocket /ws-stream-normalized error:', e);
}
finally {
// this will close underlying open WS connections
if (messages !== undefined) {
messages.return();
}
}
}
exports.streamNormalizedWS = streamNormalizedWS;
//# sourceMappingURL=streamnormalized.js.map