bitcore-node
Version:
A blockchain indexing node with extended capabilities using bitcore
194 lines • 7.51 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MergedStream = exports.ParseStream = exports.ExternalApiStream = void 0;
const axios_1 = __importDefault(require("axios"));
const stream_1 = require("stream");
const streamWithEventPipe_1 = require("../../../../utils/streamWithEventPipe");
class ExternalApiStream extends streamWithEventPipe_1.ReadableWithEventPipe {
constructor(url, headers, args) {
super({ objectMode: true });
this.url = url;
this.headers = headers;
this.cursor = null; // Start without a cursor
this.page = 0; // Start at page 0
this.results = 0; // Result count
this.limit = args?.limit; // Results limit across all pages
this.paging = args?.paging; // Total pages to retrieve
this.transform = args?.transform; // Function to transform results data
}
async _read() {
try {
// End stream if page limit is reached
if (this.paging && this.page >= this.paging) {
this.push(null);
}
const urlWithCursor = this.cursor ? `${this.url}&cursor=${this.cursor}` : this.url;
const response = await axios_1.default.get(urlWithCursor, { headers: this.headers });
if (response?.data?.result?.length > 0) {
for (const result of response.data.result) {
// End stream if result limit is reached
if (this.limit && this.results >= this.limit) {
this.push(null);
return;
}
let data = result;
// Transform data before pushing
if (this.transform) {
data = this.transform(data);
}
this.push(data);
this.results++;
}
// Update the cursor with the new value from the response
this.cursor = response.data.cursor;
// If there is no new cursor, push null to end the stream
if (!this.cursor) {
this.push(null);
}
// Page complete, increment
this.page++;
}
else {
// No more data, end the stream
this.push(null);
}
}
catch (error) {
this.emit('error', error);
}
}
// handles events emitted by the streamed response, request from client, and response to client
static onStream(stream, req, res, opts = {}) {
return new Promise((resolve, reject) => {
let closed = false;
let isFirst = true;
req.on('close', function () {
closed = true;
});
res.type('json');
res.on('close', function () {
closed = true;
});
stream.on('error', function (err) {
if (!closed) {
closed = true;
if (err.isAxiosError) {
err.log = {
url: err?.config?.url,
statusCode: err?.response?.status,
statusMsg: err?.response?.statusText,
data: err?.response?.data,
};
}
if (err.log?.data?.message?.includes('not supported')) {
res.write('[]');
res.end();
return resolve({ success: false, error: err });
}
if (!isFirst) {
// Data has already been written to the stream and status 200 headers have already been sent
// We notify and log the error instead of throwing
const errMsg = '{"error": "An error occurred during data stream"}';
if (opts.jsonl) {
res.write(`${errMsg}`);
}
else {
res.write(`,\n${errMsg}\n]`);
}
res.end();
res.destroy();
return resolve({ success: false, error: err });
}
else {
// Rejecting here allows downstream to send status 500
return reject(err);
}
}
return;
});
stream.on('data', function (data) {
if (!closed) {
// We are assuming jsonl data appended a new line upstream
if (!opts.jsonl) {
if (isFirst) {
res.write('[\n');
}
else {
res.write(',\n');
}
}
if (isFirst) {
// All cases need isFirst set correctly for proper error handling
isFirst = false;
}
if (typeof data !== 'string') {
data = JSON.stringify(data);
}
res.write(data);
}
else {
stream.destroy();
}
});
stream.on('end', function () {
if (!closed) {
closed = true;
if (!opts.jsonl) {
if (isFirst) {
// there was no data
res.write('[]');
}
else {
res.write('\n]');
}
}
res.end();
resolve({ success: true });
}
});
});
}
static mergeStreams(streams, destination) {
let activeStreams = streams.length;
for (const stream of streams) {
// Pipe each stream to the destination
stream.pipe(destination, { end: false });
stream.on('error', err => destination.emit('error', err));
stream.on('end', () => {
activeStreams--;
if (activeStreams === 0) {
// End the destination stream when all input streams are done
destination.end();
}
});
}
;
return destination;
}
}
exports.ExternalApiStream = ExternalApiStream;
class ParseStream extends stream_1.Transform {
constructor() {
super({ objectMode: true });
}
async _transform(data, _, done) {
if (typeof data === 'string') {
data = JSON.parse(data);
}
done(null, data);
}
}
exports.ParseStream = ParseStream;
class MergedStream extends streamWithEventPipe_1.TransformWithEventPipe {
constructor() {
super({ objectMode: true });
}
async _transform(data, _, done) {
done(null, data);
}
}
exports.MergedStream = MergedStream;
//# sourceMappingURL=apiStream.js.map