tardis-dev/dist/downloaddatasets.js

Convenient access to tick-level historical and real-time cryptocurrency market data via Node.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadDatasets = downloadDatasets;
const fs_extra_1 = require("fs-extra");
const p_map_1 = __importDefault(require("p-map"));
const debug_1 = require("./debug");
const handy_1 = require("./handy");
const options_1 = require("./options");
const CONCURRENCY_LIMIT = 20;
const MILLISECONDS_IN_SINGLE_DAY = 24 * 60 * 60 * 1000;
const DEFAULT_DOWNLOAD_DIR = './datasets';
const options = (0, options_1.getOptions)();
async function downloadDatasets(downloadDatasetsOptions) {
    const { exchange, dataTypes, from, to, symbols } = downloadDatasetsOptions;
    const apiKey = downloadDatasetsOptions.apiKey !== undefined ? downloadDatasetsOptions.apiKey : options.apiKey;
    const downloadDir = downloadDatasetsOptions.downloadDir !== undefined ? downloadDatasetsOptions.downloadDir : DEFAULT_DOWNLOAD_DIR;
    const format = downloadDatasetsOptions.format !== undefined ? downloadDatasetsOptions.format : 'csv';
    const getFilename = downloadDatasetsOptions.getFilename !== undefined ? downloadDatasetsOptions.getFilename : getFilenameDefault;
    const skipIfExists = downloadDatasetsOptions.skipIfExists === undefined ? true : downloadDatasetsOptions.skipIfExists;
    // in case someone provided 'api/exchange' symbol, transform it to symbol that is accepted by datasets API
    const datasetsSymbols = symbols.map((s) => s.replace(/\/|:/g, '-').toUpperCase());
    for (const symbol of datasetsSymbols) {
        for (const dataType of dataTypes) {
            const { daysCountToFetch, startDate } = getDownloadDateRange(downloadDatasetsOptions);
            const startTimestamp = new Date().valueOf();
            (0, debug_1.debug)('dataset download started for %s %s %s from %s to %s', exchange, dataType, symbol, from, to);
            if (daysCountToFetch > 1) {
                // start with downloading last day of the range, validates is API key has access to the end range of requested data
                await downloadDataSet(getDownloadOptions({
                    exchange,
                    symbol,
                    apiKey,
                    downloadDir,
                    dataType,
                    format,
                    getFilename,
                    date: (0, handy_1.addDays)(startDate, daysCountToFetch - 1)
                }), skipIfExists);
            }
            // then download the first day of the range, validates is API key has access to the start range of requested data
            await downloadDataSet(getDownloadOptions({
                exchange,
                symbol,
                apiKey,
                downloadDir,
                dataType,
                format,
                getFilename,
                date: startDate
            }), skipIfExists);
            // download the rest concurrently up to the CONCURRENCY_LIMIT
            await (0, p_map_1.default)((0, handy_1.sequence)(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
            (offset) => downloadDataSet(getDownloadOptions({
                exchange,
                symbol,
                apiKey,
                downloadDir,
                dataType,
                format,
                getFilename,
                date: (0, handy_1.addDays)(startDate, offset)
            }), skipIfExists), { concurrency: CONCURRENCY_LIMIT });
            const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000;
            (0, debug_1.debug)('dataset download finished for %s %s %s from %s to %s, time: %s seconds', exchange, dataType, symbol, from, to, elapsedSeconds);
        }
    }
}
async function downloadDataSet(downloadOptions, skipIfExists) {
    if (skipIfExists && (0, fs_extra_1.existsSync)(downloadOptions.downloadPath)) {
        (0, debug_1.debug)('dataset %s already exists, skipping download', downloadOptions.downloadPath);
    }
    else {
        return await (0, handy_1.download)(downloadOptions);
    }
}
function getDownloadOptions({ apiKey, exchange, dataType, date, symbol, format, downloadDir, getFilename }) {
    const year = date.getUTCFullYear();
    const month = (0, handy_1.doubleDigit)(date.getUTCMonth() + 1);
    const day = (0, handy_1.doubleDigit)(date.getUTCDate());
    const url = `${options.datasetsEndpoint}/${exchange}/${dataType}/${year}/${month}/${day}/${symbol}.${format}.gz`;
    const filename = getFilename({
        dataType,
        date,
        exchange,
        format,
        symbol
    });
    const downloadPath = `${downloadDir}/${filename}`;
    return {
        url,
        downloadPath,
        userAgent: options._userAgent,
        apiKey
    };
}
function getFilenameDefault({ exchange, dataType, format, date, symbol }) {
    return `${exchange}_${dataType}_${date.toISOString().split('T')[0]}_${symbol}.${format}.gz`;
}
function getDownloadDateRange({ from, to }) {
    if (!from || isNaN(Date.parse(from))) {
        throw new Error(`Invalid "from" argument: ${from}. Please provide valid date string.`);
    }
    if (!to || isNaN(Date.parse(to))) {
        throw new Error(`Invalid "to" argument: ${to}. Please provide valid date string.`);
    }
    const toDate = (0, handy_1.parseAsUTCDate)(to);
    const fromDate = (0, handy_1.parseAsUTCDate)(from);
    const daysCountToFetch = Math.floor((toDate.getTime() - fromDate.getTime()) / MILLISECONDS_IN_SINGLE_DAY);
    if (daysCountToFetch < 1) {
        throw new Error(`Invalid "to" and "from" arguments combination. Please provide "to" day that is later than "from" day.`);
    }
    return {
        startDate: fromDate,
        daysCountToFetch
    };
}
//# sourceMappingURL=downloaddatasets.js.map