dukascopy-node/dist/esm/cli/index.js

Node.js library for downloading historical market tick data for for Crypto, Stocks, ETFs, CFDs, Forex

#!/usr/bin/env node
import {
  BufferFetcher,
  CacheManager,
  VolumeUnit,
  __require,
  __spreadValues,
  formatBytes,
  generateUrls,
  getDateTimeFormatOptions,
  getFormattedDate,
  instrumentMetaData,
  normaliseDates,
  processData,
  schema,
  validateConfig,
  version
} from "../chunk-J7QH5GFA.js";

// src/cli/cli.ts
import { resolve, join } from "path";
import os from "os";

// src/cli/progress.ts
import { Bar } from "cli-progress";
var chalk = __require("chalk");
var progressBar = new Bar({
  format: "|" + chalk.green("{bar}") + "| {percentage}%",
  barCompleteChar: "\u2588",
  barIncompleteChar: "\u2591",
  hideCursor: true,
  fps: 24,
  barsize: 45
});

// src/cli/cli.ts
import { createWriteStream } from "fs";
import { ensureDir, ensureFile, stat } from "fs-extra";

// src/cli/config.ts
import { program } from "commander";
var now = "now";
var commanderSchema = program.option("-d, --debug", "Output extra debugging", false).option("-s, --silent", "Hides the search config in the CLI output", false).requiredOption("-i, --instrument <value>", "Trading instrument").requiredOption("-from, --date-from <value>", "From date (yyyy-mm-dd)").option("-to, --date-to <value>", `To date (yyyy-mm-dd or '${now}')`, now).option("-t, --timeframe <value>", "Timeframe aggregation (tick, s1, m1, m5, m15, m30, h1, h4, d1, mn1)", "d1" /* d1 */).option("-p, --price-type <value>", "Price type: (bid, ask)", "bid" /* bid */).option("-utc, --utc-offset <value>", "UTC offset in minutes", Number, 0).option("-v, --volumes", "Include volumes", false).option("-vu, --volume-units <value>", "Volume units (millions, thousands, units)", VolumeUnit.millions).option("-fl, --flats", "Include flats (0 volumes)", false).option("-f, --format <value>", "Output format (csv, json, array)", "json" /* json */).option("-dir, --directory <value>", "Download directory", "./download").option("-bs, --batch-size <value>", "Batch size of downloaded artifacts", Number, 10).option("-bp, --batch-pause <value>", "Pause between batches in ms", Number, 1e3).option("-ch, --cache", "Use cache", false).option("-chpath, --cache-path <value>", "Folder path for cache data", "./.dukascopy-cache").option("-df, --date-format <value>", "Date format", "").option("-tz, --time-zone <value>", "Timezone", "").option("-r, --retries <value>", "Number of retries for a failed artifact download", Number, 0).option("-rp, --retry-pause <value>", "Pause between retries in milliseconds", Number, 500).option("-re, --retry-on-empty", "A flag indicating whether requests with successful but empty (0 Bytes) responses should be retried. If `retries` is `0` this parameter will be ignored", false).option("-fr, --no-fail-after-retries", "A flag indicating whether the process should fail after all retries have been exhausted. If `retries` is `0` this parameter will be ignored").option("-fn, --file-name <value>", "Custom file name for the generated file", "").option("-in, --inline", "Makes files smaller in size by removing new lines in the output (works only with json and array formats)", false);
function getConfigFromCliArgs(argv) {
  const options = commanderSchema.parse(argv).opts();
  if (options.dateTo === now) {
    options.dateTo = new Date();
  }
  const cliConfig = {
    instrument: options.instrument,
    dates: {
      from: options.dateFrom,
      to: options.dateTo
    },
    timeframe: options.timeframe,
    priceType: options.priceType,
    utcOffset: options.utcOffset,
    volumes: options.volumes,
    volumeUnits: options.volumeUnits,
    ignoreFlats: !options.flats,
    dir: options.directory,
    silent: options.silent,
    format: options.format,
    batchSize: options.batchSize,
    pauseBetweenBatchesMs: options.batchPause,
    useCache: options.cache,
    cacheFolderPath: options.cachePath,
    retryCount: options.retries,
    failAfterRetryCount: options.failAfterRetries,
    retryOnEmpty: options.retryOnEmpty,
    pauseBetweenRetriesMs: options.retryPause,
    debug: options.debug,
    inline: options.inline,
    fileName: options.fileName,
    dateFormat: options.dateFormat,
    timeZone: options.timeZone
  };
  const cliSchema = __spreadValues(__spreadValues({}, schema), {
    dir: { type: "string", required: true },
    silent: { type: "boolean", required: false },
    debug: { type: "boolean", required: false },
    inline: { type: "boolean", required: false },
    fileName: { type: "string", required: false },
    dateFormat: { type: "string", required: false },
    timeZone: { type: "string", required: false }
  });
  return validateConfig(cliConfig, cliSchema);
}

// src/cli/printer.ts
var chalk2 = __require("chalk");
var log = console.log;
function printSpacer() {
  log();
}
function printDivider() {
  log(chalk2.gray("----------------------------------------------------"));
}
function printHeader(searchConfig, adjustedStartDate, adjustedEndDate) {
  const { instrument, timeframe, priceType, utcOffset, volumes, ignoreFlats, format } = searchConfig;
  const dateTimeFormatOptions = getDateTimeFormatOptions(timeframe);
  printDivider();
  log(chalk2.whiteBright("Downloading historical price data for:"));
  printDivider();
  log("Instrument:    ", chalk2.bold(chalk2.yellow(instrumentMetaData[instrument].description)));
  log("Timeframe:     ", chalk2.bold(chalk2.yellow(timeframe)));
  log("From date:     ", chalk2.bold(chalk2.yellow(getFormattedDate(adjustedStartDate, dateTimeFormatOptions))));
  log("To date:       ", chalk2.bold(chalk2.yellow(getFormattedDate(adjustedEndDate, dateTimeFormatOptions))));
  if (timeframe !== "tick") {
    log("Price type:    ", chalk2.bold(chalk2.yellow(priceType)));
  }
  log("Volumes:       ", chalk2.bold(chalk2.yellow(volumes)));
  log("UTC Offset:    ", chalk2.bold(chalk2.yellow(utcOffset)));
  log("Include flats: ", chalk2.bold(chalk2.yellow(!ignoreFlats)));
  log("Format:        ", chalk2.bold(chalk2.yellow(format)));
  printDivider();
}
function printErrors(header, errorMessage) {
  log(chalk2.redBright(header));
  [].concat(errorMessage).forEach((error) => log(chalk2.red(` > ${error}`)));
  printSpacer();
}
function printSuccess(text) {
  printDivider();
  log(chalk2.greenBright(text));
  printSpacer();
}
function printGeneral(text) {
  log(text);
  printSpacer();
}

// src/cli/cli.ts
import chalk3 from "chalk";
import debug from "debug";
import dayjs from "dayjs";
import utc from "dayjs/plugin/utc";
import tz from "dayjs/plugin/timezone";

// src/stream-writer/index.ts
import fs from "fs";
var BatchStreamWriter = class {
  constructor(options) {
    this.isFileEmpty = true;
    this.fileWriteStream = options.fileWriteStream;
    this.timeframe = options.timeframe;
    this.format = options.format;
    this.isInline = options.isInline;
    this.volumes = Boolean(options.volumes);
    this.startDateTs = options.startDateTs;
    this.endDateTs = options.endDateTs;
    this.bodyHeaders = this.initHeaders();
  }
  initHeaders() {
    const bodyHeaders = this.timeframe === "tick" /* tick */ ? ["timestamp", "askPrice", "bidPrice", "askVolume", "bidVolume"] : ["timestamp", "open", "high", "low", "close", "volume"];
    if (!this.volumes) {
      bodyHeaders.pop();
      if (this.timeframe === "tick" /* tick */) {
        bodyHeaders.pop();
      }
    }
    return bodyHeaders;
  }
  async writeBatch(batch, dateFormatter) {
    const batchWithinRange = [];
    for (let j = 0; j < batch.length; j++) {
      const item = batch[j];
      const isItemInRange = item.length > 0 && item[0] >= this.startDateTs && item[0] < this.endDateTs;
      if (isItemInRange) {
        if (dateFormatter) {
          item[0] = dateFormatter(item[0]);
        }
        batchWithinRange.push(item);
      }
    }
    for (let i = 0; i < batchWithinRange.length; i++) {
      let item = batchWithinRange[i];
      const isFirstItem = i === 0;
      const isLastItem = i === batchWithinRange.length - 1;
      const shouldOpen = isFirstItem && this.isFileEmpty;
      let body = "";
      if (this.format === "csv" /* csv */) {
        if (shouldOpen) {
          const csvHeaders = this.bodyHeaders.join(",");
          body += `${csvHeaders}
`;
        }
        body += item.join(",") + "\n";
      } else {
        if (shouldOpen) {
          body += "[" + (!this.isInline ? "\n" : "");
        }
        if (isFirstItem && !this.isFileEmpty) {
          body += "," + (!this.isInline ? "\n" : "");
        }
        if (dateFormatter && (this.format === "json" /* json */ || this.format === "array" /* array */)) {
          item[0] = `"${item[0]}"`;
        }
        if (this.format === "json" /* json */) {
          const jsonObjectBody = item.map((val, i2) => `"${this.bodyHeaders[i2]}":${val}`).join(",");
          body += `{${jsonObjectBody}}`;
        } else {
          const arrayBody = item.join(",");
          body += `[${arrayBody}]`;
        }
        body += (!isLastItem ? "," : "") + (!isLastItem && !this.isInline ? "\n" : "");
      }
      if (!body) {
        continue;
      }
      const ableToWrite = this.fileWriteStream.write(body);
      if (ableToWrite) {
        if (this.isFileEmpty) {
          this.isFileEmpty = false;
        }
      } else {
        await new Promise((resolve2) => {
          this.fileWriteStream.once("drain", resolve2);
        });
      }
    }
    return true;
  }
  async closeBatchFile() {
    if ((this.format === "json" /* json */ || this.format === "array" /* array */) && !this.isFileEmpty) {
      const body = this.isInline ? "]" : "\n]";
      const ableToWrite = this.fileWriteStream.write(body);
      if (!ableToWrite) {
        await new Promise((resolve2) => {
          this.fileWriteStream.once("drain", resolve2);
        });
      }
    }
    this.fileWriteStream.end();
    return true;
  }
};

// src/utils/formatTimeDuration.ts
function formatTimeDuration(durationMs) {
  if (durationMs < 1e3) {
    return `${durationMs}ms`;
  } else if (durationMs < 6e4) {
    return `${(durationMs / 1e3).toFixed(1)}s`;
  } else if (durationMs < 36e5) {
    const min = Math.floor(durationMs / 6e4);
    const sec = Math.floor((durationMs - min * 6e4) / 1e3);
    return `${min}m ${sec}s`;
  } else {
    const hours = Math.floor(durationMs / 36e5);
    const min = Math.floor((durationMs - hours * 36e5) / 6e4);
    const sec = Math.floor((durationMs - hours * 36e5 - min * 6e4) / 1e3);
    return `${hours}h ${min}m ${sec}s`;
  }
}

// src/cli/cli.ts
dayjs.extend(utc);
dayjs.extend(tz);
var DEBUG_NAMESPACE = "dukascopy-node:cli";
async function run(argv) {
  const { input, isValid, validationErrors } = getConfigFromCliArgs(argv);
  let {
    instrument,
    dates: { from: fromDate, to: toDate },
    timeframe,
    priceType,
    utcOffset,
    volumes,
    volumeUnits,
    ignoreFlats,
    format,
    batchSize,
    pauseBetweenBatchesMs,
    useCache,
    cacheFolderPath,
    dir,
    silent,
    debug: isDebugActive,
    inline,
    retryCount,
    failAfterRetryCount,
    retryOnEmpty,
    pauseBetweenRetriesMs,
    fileName: customFileName,
    dateFormat,
    timeZone
  } = input;
  if (isDebugActive) {
    debug.enable(`${DEBUG_NAMESPACE}:*`);
  } else {
    if (process.env.DEBUG) {
      isDebugActive = true;
      debug.enable(process.env.DEBUG);
    }
  }
  const downloadStartTs = Date.now();
  try {
    debug(`${DEBUG_NAMESPACE}:version`)(version);
    debug(`${DEBUG_NAMESPACE}:nodejs`)(process.version);
    debug(`${DEBUG_NAMESPACE}:os`)(`${os.type()}, ${os.release()} (${os.platform()})`);
    debug(`${DEBUG_NAMESPACE}:config`)("%O", {
      input,
      isValid,
      validationErrors
    });
    if (isValid) {
      const [startDate, endDate] = normaliseDates({
        instrument,
        startDate: fromDate,
        endDate: toDate,
        timeframe,
        utcOffset
      });
      const fileExtension = format === "csv" /* csv */ ? "csv" /* csv */ : "json" /* json */;
      const dateRangeStr = [startDate, endDate].map((date) => {
        let cutoff = 10;
        const hasHours = date.getUTCHours() !== 0;
        const hasMinutes = date.getUTCMinutes() !== 0;
        if (hasHours) {
          cutoff = 13;
        }
        if (hasMinutes) {
          cutoff = 16;
        }
        return date.toISOString().slice(0, cutoff);
      }).join("-");
      const fileName = customFileName ? `${customFileName}.${fileExtension}` : `${instrument}-${timeframe}${timeframe === "tick" ? "" : "-" + priceType}-${dateRangeStr}.${fileExtension}`;
      const folderPath = resolve(process.cwd(), dir);
      const filePath = resolve(folderPath, fileName);
      if (!isDebugActive) {
        silent ? printDivider() : printHeader(input, startDate, endDate);
      }
      const urls = generateUrls({ instrument, timeframe, priceType, startDate, endDate });
      debug(`${DEBUG_NAMESPACE}:urls`)(`Generated ${urls.length} urls`);
      debug(`${DEBUG_NAMESPACE}:urls`)(`%O`, urls);
      let step = 0;
      if (!isDebugActive) {
        progressBar.start(urls.length, step);
      }
      await ensureDir(folderPath);
      const fileWriteStream = createWriteStream(filePath, { flags: "w+" });
      fileWriteStream.on("finish", async () => {
        const downloadEndTs = Date.now();
        if (!isDebugActive) {
          progressBar.stop();
        }
        const relativeFilePath = join(dir, fileName);
        await ensureFile(filePath);
        const { size } = await stat(filePath);
        printSuccess(`\u221A File saved: ${chalk3.bold(relativeFilePath)} (${formatBytes(size)})`);
        printGeneral(`Download time: ${formatTimeDuration(downloadEndTs - downloadStartTs)}`);
      });
      const batchStreamWriter = new BatchStreamWriter({
        fileWriteStream,
        timeframe,
        format,
        isInline: inline,
        volumes,
        startDateTs: +startDate,
        endDateTs: +endDate
      });
      const bufferFetcher = new BufferFetcher({
        batchSize,
        pauseBetweenBatchesMs,
        cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
        retryCount,
        retryOnEmpty,
        failAfterRetryCount,
        pauseBetweenRetriesMs,
        onItemFetch: (url, buffer, isCacheHit) => {
          debug(`${DEBUG_NAMESPACE}:fetcher`)(url, `| ${formatBytes(buffer.length)} |`, `${isCacheHit ? "cache" : "network"}`);
          if (!isDebugActive) {
            step += 1;
            progressBar.update(step);
          }
        },
        onBatchFetch: async (bufferObjects, isLastBatch) => {
          const filteredBatchData = [];
          for (let j = 0, m = bufferObjects.length; j < m; j++) {
            if (bufferObjects[j].buffer.length > 0) {
              filteredBatchData.push(bufferObjects[j]);
            }
          }
          if (filteredBatchData.length) {
            const processedBatch = processData({
              instrument,
              requestedTimeframe: timeframe,
              bufferObjects: filteredBatchData,
              priceType,
              volumes,
              volumeUnits,
              ignoreFlats
            });
            await batchStreamWriter.writeBatch(processedBatch, dateFormat ? (timeStamp) => {
              if (dateFormat === "iso") {
                return new Date(timeStamp).toISOString();
              }
              if (timeZone) {
                return dayjs(timeStamp).tz(timeZone).format(dateFormat);
              }
              return dayjs(timeStamp).utc().format(dateFormat);
            } : void 0);
          }
          if (isLastBatch) {
            await batchStreamWriter.closeBatchFile();
          }
        }
      });
      await bufferFetcher.fetch_optimized(urls);
    } else {
      printErrors("Search config invalid:", validationErrors.map((err) => (err == null ? void 0 : err.message) || ""));
      process.exit(1);
    }
  } catch (err) {
    const errorMsg = err instanceof Error ? err.message : JSON.stringify(err);
    printErrors("\nSomething went wrong:", errorMsg);
    process.exit(1);
  }
}

// src/cli/index.ts
(async () => {
  await run(process.argv);
})();