ak-fetch
Version:
Production-ready HTTP client for bulk operations with connection pooling, exponential backoff, streaming, and comprehensive error handling
1,352 lines (1,339 loc) • 141 kB
JavaScript
#! /usr/bin/env node
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __esm = (fn, res) => function __init() {
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
};
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// lib/logger.js
function createLogger(options = {}) {
return new AkLogger(options);
}
var import_readline, import_ak_tools, AkLogger;
var init_logger = __esm({
"lib/logger.js"() {
import_readline = __toESM(require("readline"), 1);
import_ak_tools = require("ak-tools");
AkLogger = class _AkLogger {
constructor(options = {}) {
this.verbose = options.verbose !== false;
this.startTime = null;
this.lastProgressUpdate = 0;
this.progressUpdateInterval = options.progressUpdateInterval || 250;
this.showThroughput = options.showThroughput !== false;
this.showMemory = options.showMemory || false;
this.progressBarWidth = options.progressBarWidth || 30;
this.logPrefix = options.logPrefix || "\u{1F680}";
this.progressActive = false;
this.originalLog = console.log;
this.originalError = console.error;
this.originalWarn = console.warn;
this.log = this.log.bind(this);
this.error = this.error.bind(this);
this.warn = this.warn.bind(this);
this.progress = this.progress.bind(this);
}
/**
* Start a new operation with initial logging
*
* @description
* Initializes timing and displays operation start message with configuration.
* Sets up progress tracking and logs relevant configuration details.
*
* @param {string} message - Operation description to display
* @description Brief description of what operation is starting
*
* @param {Object} [config={}] - Configuration details to display
* @param {string} [config.url] - Target URL
* @param {string} [config.method] - HTTP method
* @param {number} [config.batchSize] - Batch size
* @param {number} [config.concurrency] - Concurrency level
* @param {number} [config.retries] - Retry attempts
* @param {number} [config.timeout] - Request timeout
*
* @example
* logger.start('Processing bulk upload', {
* url: 'https://api.example.com/bulk',
* method: 'POST',
* batchSize: 100,
* concurrency: 10
* });
*
* @since 2.0.0
*/
start(message, config = {}) {
this.startTime = Date.now();
this.lastProgressUpdate = 0;
if (!this.verbose) return;
console.log(`
${this.logPrefix} ${message}`);
if (config && Object.keys(config).length > 0) {
this.logConfig(config);
}
console.log("");
}
/**
* Log configuration details in a friendly format
*
* @description
* Displays configuration settings in a clean, readable format.
* Shows key operational parameters and enabled features.
*
* @param {Object} config - Configuration object to display
* @param {string} [config.url] - Target URL
* @param {string} [config.method] - HTTP method
* @param {number} [config.batchSize] - Items per batch
* @param {number} [config.concurrency] - Concurrent requests
* @param {number} [config.retries] - Retry attempts
* @param {number} [config.timeout] - Request timeout
* @param {Array|string} [config.data] - Data source
*
* @example
* logger.logConfig({
* url: 'https://api.example.com',
* method: 'POST',
* batchSize: 50,
* concurrency: 5,
* enableCookies: true
* });
* // Outputs formatted configuration display
*
* @since 2.0.0
*/
logConfig(config) {
if (!this.verbose) return;
const {
url,
method = "POST",
batchSize,
concurrency,
retries,
timeout,
data,
...otherConfig
} = config;
console.log("\u{1F4CB} Configuration:");
console.log(` URL: ${url}`);
console.log(` Method: ${method.toUpperCase()}`);
if (Array.isArray(data)) {
console.log(` Records: ${(0, import_ak_tools.comma)(data.length)}`);
} else if (typeof data === "string") {
console.log(` Data Source: ${data}`);
}
if (batchSize) {
console.log(` Batch Size: ${(0, import_ak_tools.comma)(batchSize)}`);
}
if (concurrency) {
console.log(` Concurrency: ${concurrency}`);
}
if (retries !== void 0) {
console.log(` Retries: ${retries === null ? "Fire-and-forget" : retries}`);
}
if (timeout) {
console.log(` Timeout: ${this.formatDuration(timeout)}`);
}
const interestingKeys = ["enableCookies", "enableConnectionPooling", "useStaticRetryDelay"];
const additionalConfig = {};
interestingKeys.forEach((key) => {
if (otherConfig[key] !== void 0) {
additionalConfig[key] = otherConfig[key];
}
});
if (Object.keys(additionalConfig).length > 0) {
console.log(` Features: ${Object.entries(additionalConfig).filter(([, value]) => value).map(([key]) => this.camelToTitle(key)).join(", ")}`);
}
}
/**
* Update progress display with current statistics
*
* @description
* Updates the progress display with current statistics including completion
* percentage, throughput, and ETA. Throttles updates for performance.
*
* @param {number} completed - Number of completed items
* @description Usually represents completed batches or requests
*
* @param {number} [total=0] - Total number of items to process
* @description When 0, shows completed count without percentage
*
* @param {number} [records=0] - Number of individual records processed
* @description Distinct from batches, represents actual data items
*
* @param {Object} [options={}] - Additional progress options
* @description Reserved for future progress display options
*
* @example
* // Basic progress with percentage
* logger.progress(75, 100, 7500);
* // Displays: [████████████████████░░░░░░░░░░] 75% | 75/100 batches | 7,500 records | 25 req/s
*
* @example
* // Progress without total (streaming mode)
* logger.progress(150, 0, 15000);
* // Displays: [████████████████████████████████] 0% | 150 requests | 15,000 records | 30 req/s
*
* @since 2.0.0
*/
progress(completed, total = 0, records = 0, options = {}) {
if (!this.verbose) return;
const now = Date.now();
if (now - this.lastProgressUpdate < this.progressUpdateInterval) {
return;
}
this.lastProgressUpdate = now;
this.progressActive = true;
import_readline.default.cursorTo(process.stdout, 0);
import_readline.default.clearLine(process.stdout, 0);
const percent = total > 0 ? Math.floor(completed / total * 100) : 0;
const progressBar = this.createProgressBar(percent);
let message = `${progressBar} ${percent}%`;
if (total > 0) {
message += ` | ${(0, import_ak_tools.comma)(completed)}/${(0, import_ak_tools.comma)(total)} batches`;
} else {
message += ` | ${(0, import_ak_tools.comma)(completed)} requests`;
}
if (records > 0) {
message += ` | ${(0, import_ak_tools.comma)(records)} records`;
}
if (this.showThroughput && this.startTime && completed > 0) {
const elapsed = (now - this.startTime) / 1e3;
const rps = Math.floor(completed / elapsed);
if (rps > 0) {
message += ` | ${(0, import_ak_tools.comma)(rps)} req/s`;
}
if (records > 0) {
const recordsPerSec = Math.floor(records / elapsed);
if (recordsPerSec > 0) {
message += ` (${(0, import_ak_tools.comma)(recordsPerSec)} rec/s)`;
}
}
}
if (this.showMemory) {
const memUsage = process.memoryUsage();
const heapMB = Math.round(memUsage.heapUsed / 1024 / 1024);
message += ` | ${heapMB}MB`;
}
if (this.startTime && total > 0 && completed > 0 && percent < 100) {
const elapsed = now - this.startTime;
const rate = completed / elapsed;
const remaining = total - completed;
const eta = remaining / rate;
if (eta > 0 && eta < 864e5) {
message += ` | ETA: ${this.formatDuration(eta)}`;
}
}
process.stdout.write(` ${message}`);
}
/**
* Complete progress display and show final statistics
*
* @description
* Clears progress line and displays final operation statistics.
* Shows completion status, timing, throughput, and error counts.
*
* @param {Object} [results={}] - Final operation results
* @param {number} [results.reqCount] - Total requests made
* @param {number} [results.rowCount] - Total records processed
* @param {number} [results.duration] - Operation duration in ms
* @param {number} [results.rps] - Requests per second
* @param {number} [results.errors] - Number of errors
* @param {Object} [results.stats] - Memory statistics
*
* @example
* logger.complete({
* reqCount: 100,
* rowCount: 10000,
* duration: 30000,
* rps: 3.33,
* errors: 2
* });
* // Displays completion summary with all metrics
*
* @since 2.0.0
*/
complete(results = {}) {
if (!this.verbose) return;
import_readline.default.cursorTo(process.stdout, 0);
import_readline.default.clearLine(process.stdout, 0);
const {
reqCount = 0,
rowCount = 0,
duration = 0,
rps = 0,
errors = 0
} = results;
const emoji = errors > 0 ? "\u26A0\uFE0F" : "\u2705";
console.log(`${emoji} Completed: ${(0, import_ak_tools.comma)(reqCount)} requests`);
if (rowCount > 0) {
console.log(` \u{1F4CA} Processed: ${(0, import_ak_tools.comma)(rowCount)} records`);
}
if (duration > 0) {
console.log(` \u23F1\uFE0F Duration: ${this.formatDuration(duration)}`);
}
if (rps > 0) {
console.log(` \u{1F684} Throughput: ${(0, import_ak_tools.comma)(rps)} requests/second`);
if (rowCount > 0) {
const recordsPerSec = Math.floor(rowCount / (duration / 1e3));
console.log(` \u{1F4C8} Records/sec: ${(0, import_ak_tools.comma)(recordsPerSec)}`);
}
}
if (errors > 0) {
console.log(` \u274C Errors: ${(0, import_ak_tools.comma)(errors)}`);
}
if (this.showMemory && results.stats) {
console.log(` \u{1F4BE} Memory: ${results.stats.heapUsed}MB heap, ${results.stats.rss}MB RSS`);
}
console.log("");
}
/**
* Log general messages (respects verbose setting)
*
* @description
* Logs messages only when verbose mode is enabled. Uses standard console.log
* behavior for formatting and output.
*
* @param {...any} args - Arguments to log (same as console.log)
* @description Supports all console.log argument types and formatting
*
* @example
* logger.log('Processing started');
* logger.log('Found %d records in %s', count, filename);
* logger.log({ config: settings });
*
* @since 2.0.0
*/
log(...args) {
if (!this.verbose) return;
this.originalLog(...args);
}
/**
* Log error messages (always shown regardless of verbose setting)
*
* @description
* Logs error messages with error emoji prefix. Always displayed regardless
* of verbose setting since errors are critical information.
*
* @param {...any} args - Arguments to log (same as console.error)
* @description Supports all console.error argument types and formatting
*
* @example
* logger.error('Request failed:', error.message);
* logger.error('Network error: %s', networkError);
*
* @since 2.0.0
*/
error(...args) {
this.originalError("\u274C", ...args);
}
/**
* Log warning messages (always shown regardless of verbose setting)
*
* @description
* Logs warning messages with warning emoji prefix. Always displayed
* regardless of verbose setting for important notifications.
*
* @param {...any} args - Arguments to log (same as console.warn)
* @description Supports all console.warn argument types and formatting
*
* @example
* logger.warn('Retrying request due to timeout');
* logger.warn('Rate limit approaching: %d/%d', current, limit);
*
* @since 2.0.0
*/
warn(...args) {
this.originalWarn("\u26A0\uFE0F", ...args);
}
/**
* Log informational messages with icon
*
* @description
* Logs informational messages with info emoji prefix. Only displayed
* when verbose mode is enabled.
*
* @param {...any} args - Arguments to log (same as console.log)
* @description Supports all console.log argument types and formatting
*
* @example
* logger.info('Connected to database');
* logger.info('Cache hit rate: %d%%', hitRate);
*
* @since 2.0.0
*/
info(...args) {
if (!this.verbose) return;
if (this.progressActive) {
import_readline.default.cursorTo(process.stdout, 0);
import_readline.default.clearLine(process.stdout, 0);
this.originalLog("\u2139\uFE0F", ...args);
this.progressActive = false;
} else {
this.originalLog("\u2139\uFE0F", ...args);
}
}
/**
* Log success messages with icon
*
* @description
* Logs success messages with success emoji prefix. Only displayed
* when verbose mode is enabled.
*
* @param {...any} args - Arguments to log (same as console.log)
* @description Supports all console.log argument types and formatting
*
* @example
* logger.success('Upload completed successfully');
* logger.success('Processed %d records', recordCount);
*
* @since 2.0.0
*/
success(...args) {
if (!this.verbose) return;
this.originalLog("\u2705", ...args);
}
/**
* Log file operation messages
*
* @description
* Logs file operation messages with file emoji prefix. Shows operation
* type, filename, and optional format information.
*
* @param {string} operation - Operation type (Reading, Writing, etc.)
* @description Capitalized operation description like 'Writing', 'Reading'
*
* @param {string} filename - File name or path
* @description Full or relative path to the file being operated on
*
* @param {string} [format=''] - File format description
* @description Optional format like 'json', 'csv', 'ndjson'
*
* @example
* logger.fileOperation('Writing', './results.json', 'json');
* // Output: 📁 Writing: ./results.json (json)
*
* @example
* logger.fileOperation('Reading', '/data/input.csv');
* // Output: 📁 Reading: /data/input.csv
*
* @since 2.0.0
*/
fileOperation(operation, filename, format = "") {
if (!this.verbose) return;
const formatStr = format ? ` (${format})` : "";
this.originalLog(`\u{1F4C1} ${operation}: ${filename}${formatStr}`);
}
/**
* Create ASCII progress bar
*
* @description
* Generates an ASCII progress bar using block characters. Creates visual
* representation of completion percentage with filled and empty sections.
*
* @param {number} percent - Percentage complete (0-100)
* @description Must be between 0 and 100, values outside range may produce unexpected results
*
* @returns {string} ASCII progress bar string
* @description Formatted progress bar like '[████████████░░░░░░░░░░░░░░░░░░]'
*
* @example
* logger.createProgressBar(75);
* // Returns: '[██████████████████████░░░░░░░░]'
*
* @example
* logger.createProgressBar(0);
* // Returns: '[░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]'
*
* @since 2.0.0
*/
createProgressBar(percent) {
const filled = Math.floor(percent / 100 * this.progressBarWidth);
const empty = this.progressBarWidth - filled;
const filledBar = "\u2588".repeat(filled);
const emptyBar = "\u2591".repeat(empty);
return `[${filledBar}${emptyBar}]`;
}
/**
* Format duration in milliseconds to human-readable string
*
* @description
* Converts millisecond durations into human-readable format with appropriate
* units. Automatically selects the most suitable unit (ms, s, m, h) based on duration.
*
* @param {number} ms - Duration in milliseconds
* @description Must be a non-negative number
*
* @returns {string} Formatted duration string
* @description Like '500ms', '2.5s', '3m 45s', '1h 30m'
*
* @example
* logger.formatDuration(500); // '500ms'
* logger.formatDuration(2500); // '2.5s'
* logger.formatDuration(90000); // '1m 30s'
* logger.formatDuration(3661000); // '1h 1m'
*
* @since 2.0.0
*/
formatDuration(ms) {
if (ms < 1e3) {
return `${Math.round(ms)}ms`;
}
const seconds = ms / 1e3;
if (seconds < 60) {
return `${seconds.toFixed(1)}s`;
}
const minutes = Math.floor(seconds / 60);
const remainingSeconds = Math.floor(seconds % 60);
if (minutes < 60) {
return `${minutes}m ${remainingSeconds}s`;
}
const hours = Math.floor(minutes / 60);
const remainingMinutes = minutes % 60;
return `${hours}h ${remainingMinutes}m`;
}
/**
* Convert camelCase to Title Case
*
* @description
* Converts camelCase strings to Title Case for display purposes.
* Useful for converting configuration property names into readable labels.
*
* @param {string} str - camelCase string to convert
* @description String in camelCase format like 'enableCookies'
*
* @returns {string} Title Case string
* @description Converted string like 'Enable Cookies'
*
* @example
* logger.camelToTitle('enableCookies'); // 'Enable Cookies'
* logger.camelToTitle('useStaticRetryDelay'); // 'Use Static Retry Delay'
* logger.camelToTitle('maxRetries'); // 'Max Retries'
*
* @since 2.0.0
*/
camelToTitle(str) {
return str.replace(/([A-Z])/g, " $1").replace(/^./, (str2) => str2.toUpperCase()).trim();
}
/**
* Temporarily disable logging
*
* @description
* Disables verbose logging output. Error and warning messages will still
* be displayed. Can be re-enabled with unsilence().
*
* @example
* logger.silence();
* logger.log('This will not be displayed');
* logger.error('This will still be displayed');
*
* @since 2.0.0
*/
silence() {
this.verbose = false;
}
/**
* Re-enable logging
*
* @description
* Re-enables verbose logging output that was previously disabled with silence().
* Restores normal logging behavior.
*
* @example
* logger.silence();
* logger.unsilence();
* logger.log('This will be displayed again');
*
* @since 2.0.0
*/
unsilence() {
this.verbose = true;
}
/**
* Check if logger is in verbose mode
*
* @description
* Returns the current verbose state of the logger. Useful for conditional
* logging logic in calling code.
*
* @returns {boolean} True if verbose mode is enabled
* @description False if logging has been silenced or verbose was set to false
*
* @example
* if (logger.isVerbose()) {
* const details = generateDetailedReport();
* logger.log('Detailed report:', details);
* }
*
* @since 2.0.0
*/
isVerbose() {
return this.verbose;
}
/**
* Create a child logger with inherited settings
*
* @description
* Creates a new logger instance that inherits settings from the parent.
* Allows overriding specific options while maintaining other settings.
*
* @param {Object} [options={}] - Options to override from parent
* @param {boolean} [options.verbose] - Override verbose setting
* @param {boolean} [options.showThroughput] - Override throughput display
* @param {boolean} [options.showMemory] - Override memory display
* @param {number} [options.progressBarWidth] - Override progress bar width
* @param {string} [options.logPrefix] - Override log prefix emoji
*
* @returns {AkLogger} New logger instance with inherited settings
* @description Independent logger that can be configured separately
*
* @example
* const parentLogger = new AkLogger({ verbose: true, showMemory: true });
* const childLogger = parentLogger.child({ logPrefix: '📊' });
* // Child inherits verbose: true, showMemory: true but uses different prefix
*
* @since 2.0.0
*/
child(options = {}) {
return new _AkLogger({
verbose: this.verbose,
showThroughput: this.showThroughput,
showMemory: this.showMemory,
progressBarWidth: this.progressBarWidth,
logPrefix: this.logPrefix,
...options
});
}
};
}
});
// cli.js
async function cliParams() {
const args = (0, import_yargs.default)(process.argv.splice(2)).scriptName("ak-fetch").usage(`${welcome}
usage:
npx $0 [data] [options]
examples:
# Basic batch processing
npx $0 ./data.json --url https://api.example.com --batchSize 50
# High-performance streaming
npx $0 ./events.jsonl --url https://api.example.com/events --batchSize 1000 --concurrency 20 --enableConnectionPooling
# Multiple HTTP methods
npx $0 ./users.json --url https://api.example.com/users --method PUT --enableCookies
# Memory-efficient large files
npx $0 ./massive-dataset.jsonl --url https://api.example.com/bulk --maxResponseBuffer 100 --storeResponses false
# Dynamic authentication
npx $0 ./data.json --url https://api.example.com/secure --shellCommand 'aws sts get-session-token --query Credentials.SessionToken --output text'
# Inline data
npx $0 --payload '[{"id": 1, "name": "test"}]' --url https://api.example.com --verbose
DOCS: https://github.com/ak--47/ak-fetch`).command("$0", "bulk fetch calls", () => {
}).option("url", {
demandOption: false,
describe: "Target API endpoint URL",
type: "string"
}).option("method", {
demandOption: false,
describe: "HTTP method (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS)",
type: "string",
default: "POST",
choices: ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"]
}).option("batch_size", {
alias: "batchSize",
demandOption: false,
describe: "Records per HTTP request (0 disables batching)",
type: "number",
default: 2
}).option("concurrency", {
demandOption: false,
describe: "Maximum concurrent requests",
type: "number",
default: 10
}).option("max_tasks", {
alias: "maxTasks",
demandOption: false,
describe: "Max queued tasks before pausing stream",
type: "number",
default: 25
}).option("delay", {
demandOption: false,
describe: "Delay between requests in milliseconds",
type: "number",
default: 0
}).option("retries", {
demandOption: false,
describe: "Max retry attempts (null for fire-and-forget)",
type: "number",
default: 3
}).option("retry_delay", {
alias: "retryDelay",
demandOption: false,
describe: "Base retry delay in milliseconds",
type: "number",
default: 1e3
}).option("retry_on", {
alias: "retryOn",
demandOption: false,
describe: "HTTP status codes to retry on (JSON array)",
type: "string",
default: "[408,429,500,502,503,504,520,521,522,523,524]"
}).option("use_static_retry_delay", {
alias: "useStaticRetryDelay",
demandOption: false,
describe: "Use fixed delays instead of exponential backoff",
type: "boolean",
default: false
}).option("timeout", {
demandOption: false,
describe: "Request timeout in milliseconds",
type: "number",
default: 6e4
}).option("dry_run", {
alias: "dryRun",
demandOption: false,
default: false,
describe: "Test mode: simulate requests without making them",
type: "boolean"
}).option("curl", {
demandOption: false,
default: false,
describe: "Generate curl commands instead of making requests",
type: "boolean"
}).option("show_data", {
alias: "showData",
demandOption: false,
default: false,
describe: "Show first 100 transformed records in dry-run mode (useful with --preset)",
type: "boolean"
}).option("show_sample", {
alias: "showSample",
demandOption: false,
default: false,
describe: "Show first 3 transformed records in dry-run mode",
type: "boolean"
}).option("no_batch", {
alias: "noBatch",
demandOption: false,
describe: "Send as single request without batching",
type: "boolean",
default: false
}).option("log_file", {
alias: "logFile",
demandOption: false,
describe: "Save responses to file",
type: "string"
}).option("format", {
demandOption: false,
describe: "Output format for log files (auto-detected from file extension if not specified)",
type: "string",
choices: ["json", "csv", "ndjson"]
}).option("verbose", {
demandOption: false,
default: true,
describe: "Enable progress display and detailed logging",
type: "boolean"
}).option("response_headers", {
alias: "responseHeaders",
demandOption: false,
describe: "Include response headers in output",
type: "boolean",
default: false
}).options("search_params", {
alias: "searchParams",
demandOption: false,
default: "{}",
describe: 'URL query parameters as JSON: {"key": "value"}',
type: "string"
}).options("body_params", {
alias: "bodyParams",
demandOption: false,
default: "{}",
describe: "Additional body parameters as JSON",
type: "string"
}).options("headers", {
demandOption: false,
default: "{}",
describe: 'HTTP headers as JSON: {"Authorization": "Bearer xxx"}',
type: "string"
}).options("payload", {
demandOption: false,
describe: "Data to send as JSON (alternative to file argument)",
type: "string"
}).option("enable_connection_pooling", {
alias: "enableConnectionPooling",
demandOption: false,
describe: "Enable HTTP connection pooling for performance",
type: "boolean",
default: true
}).option("keep_alive", {
alias: "keepAlive",
demandOption: false,
describe: "Keep TCP connections alive",
type: "boolean",
default: true
}).option("max_response_buffer", {
alias: "maxResponseBuffer",
demandOption: false,
describe: "Maximum responses kept in memory (circular buffer)",
type: "number",
default: 1e3
}).option("max_memory_usage", {
alias: "maxMemoryUsage",
demandOption: false,
describe: "Memory limit in bytes",
type: "number"
}).option("force_gc", {
alias: "forceGC",
demandOption: false,
describe: "Force garbage collection after batches",
type: "boolean",
default: false
}).option("high_water_mark", {
alias: "highWaterMark",
demandOption: false,
describe: "Stream buffer size in bytes",
type: "number",
default: 16384
}).option("enable_cookies", {
alias: "enableCookies",
demandOption: false,
describe: "Enable automatic cookie handling",
type: "boolean",
default: false
}).option("store_responses", {
alias: "storeResponses",
demandOption: false,
describe: "Store responses in memory",
type: "boolean",
default: true
}).option("clone", {
demandOption: false,
describe: "Clone data before transformation",
type: "boolean",
default: false
}).option("debug", {
demandOption: false,
describe: "Enable debug mode with detailed error info",
type: "boolean",
default: false
}).option("shell_command", {
alias: "shellCommand",
demandOption: false,
describe: "Shell command for dynamic header generation",
type: "string"
}).option("shell_header", {
alias: "shellHeader",
demandOption: false,
describe: "Header name for shell command output",
type: "string",
default: "Authorization"
}).option("shell_prefix", {
alias: "shellPrefix",
demandOption: false,
describe: "Prefix for shell command header value",
type: "string",
default: "Bearer"
}).option("preset", {
demandOption: false,
describe: "Apply vendor-specific data transformation preset",
type: "string",
choices: ["mixpanel", "amplitude", "pendo"]
}).help().wrap(null).argv;
if (args._.length === 0 && !args.payload) {
if (args.method !== "GET" && args.method !== "HEAD" && args.method !== "OPTIONS") {
throw new Error("No data provided. Please specify a file or use --payload to provide inline data.");
}
}
if (!args.url) {
throw new Error("URL is required. Use --url <endpoint> to specify the target API endpoint.");
}
if (args.headers) {
args.headers = parse(args.headers);
}
if (args.search_params) args.searchParams = parse(args.search_params);
if (args.body_params) args.bodyParams = parse(args.body_params);
if (args.retry_on) args.retryOn = parse(args.retry_on);
if (args.payload) args.data = parse(args.payload);
if (args.curl) args.dryRun = "curl";
else if (args.dry_run) args.dryRun = true;
else args.dryRun = false;
if (args.shell_command) {
args.shell = {
// @ts-ignore
command: args.shell_command,
// @ts-ignore
header: args.shell_header,
// @ts-ignore
prefix: args.shell_prefix
};
}
if (args.retries === "null" || args.retries === null) args.retries = null;
if (args.log_file && !args.format) {
const ext = args.log_file.toLowerCase().split(".").pop();
if (ext === "ndjson" || ext === "jsonl") {
args.format = "ndjson";
} else if (ext === "csv") {
args.format = "csv";
} else {
args.format = "json";
}
}
const file = args._[0];
if (file) {
try {
args.data = file;
} catch (error) {
const logger = createLogger({ verbose: true });
logger.error(`Failed to process file: ${file}`, error.message);
process.exit(1);
}
}
if (!args.data && args.method !== "GET" && args.method !== "HEAD" && args.method !== "OPTIONS") {
throw new Error("No data provided for " + args.method + " request");
}
delete args._;
delete args.$0;
delete args.shell_command;
delete args.shell_header;
delete args.shell_prefix;
delete args.retry_on;
delete args.search_params;
delete args.body_params;
return args;
}
function parse(val, defaultVal = void 0) {
if (typeof val === "string") {
try {
val = JSON.parse(val);
} catch (firstError) {
try {
if (typeof val === "string") val = JSON.parse(val?.replace(/'/g, '"'));
} catch (secondError) {
if (this.verbose) console.log(`error parsing tags: ${val}
tags must be valid JSON`);
val = defaultVal;
}
}
}
if (Object.keys(val).length === 0) return defaultVal;
return val;
}
var import_yargs, import_fs, import_ak_tools2, import_meta, packageJson, version, hero, banner, welcome, cli_default;
var init_cli = __esm({
"cli.js"() {
import_yargs = __toESM(require("yargs"), 1);
import_fs = require("fs");
import_ak_tools2 = __toESM(require("ak-tools"), 1);
init_logger();
import_meta = {};
packageJson = JSON.parse((0, import_fs.readFileSync)("./package.json", "utf8"));
({ version } = packageJson);
hero = String.raw`
░▒▓██████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓████████▓▒░▒▓████████▓▒░▒▓████████▓▒░▒▓██████▓▒░░▒▓█▓▒░░▒▓█▓▒░
░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░
░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░
░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓████████▓▒░
░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░
░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░
░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓████████▓▒░ ░▒▓█▓▒░ ░▒▓██████▓▒░░▒▓█▓▒░░▒▓█▓▒░
`;
banner = `... production-ready HTTP client for bulk operations (v${version || 2})
\u{1F680} High Performance \u2022 \u{1F504} Smart Retries \u2022 \u{1F4BE} Memory Efficient \u2022 \u{1F512} Production Ready
by AK (ak@mixpanel.com)
`;
welcome = hero.concat("\n").concat(banner);
cliParams.welcome = welcome;
if (import_meta.url === `file://${process.argv[1]}`) {
(async () => {
try {
const { default: akFetch } = await Promise.resolve().then(() => (init_index(), index_exports));
const config = await cliParams();
const result = await akFetch(config);
if (result && typeof result === "object") {
process.exit(0);
}
} catch (error) {
console.error("Error:", error.message);
process.exit(1);
}
})();
}
cli_default = cliParams;
}
});
// lib/errors.js
var AkFetchError, NetworkError, TimeoutError, RetryError, ValidationError, RateLimitError, ConfigurationError, SSLError, MemoryError;
var init_errors = __esm({
"lib/errors.js"() {
AkFetchError = class extends Error {
constructor(message, options = {}) {
super(message);
this.name = this.constructor.name;
this.code = options.code;
this.statusCode = options.statusCode;
this.url = options.url;
this.method = options.method;
this.retryCount = options.retryCount || 0;
this.timestamp = (/* @__PURE__ */ new Date()).toISOString();
Error.captureStackTrace(this, this.constructor);
}
toJSON() {
return {
name: this.name,
message: this.message,
type: this.type,
code: this.code,
statusCode: this.statusCode,
url: this.url,
method: this.method,
retryCount: this.retryCount,
timestamp: this.timestamp
};
}
};
NetworkError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "NETWORK_ERROR";
}
};
TimeoutError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "TIMEOUT_ERROR";
this.timeout = options.timeout;
}
};
RetryError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "RETRY_ERROR";
this.maxRetries = options.maxRetries;
this.lastError = options.lastError;
}
};
ValidationError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "VALIDATION_ERROR";
this.field = options.field;
this.value = options.value;
}
};
RateLimitError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "RATE_LIMIT_ERROR";
this.retryAfter = options.retryAfter;
this.limit = options.limit;
this.remaining = options.remaining;
}
};
ConfigurationError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "CONFIGURATION_ERROR";
this.parameter = options.parameter;
}
};
SSLError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "SSL_ERROR";
this.certificate = options.certificate;
}
};
MemoryError = class extends AkFetchError {
constructor(message, options = {}) {
super(message, options);
this.type = "MEMORY_ERROR";
this.memoryUsage = options.memoryUsage;
this.limit = options.limit;
}
};
}
});
// lib/retry-strategy.js
var RetryStrategy, retry_strategy_default;
var init_retry_strategy = __esm({
"lib/retry-strategy.js"() {
init_errors();
RetryStrategy = class {
/**
* Create a new retry strategy
*
* @param {Object} [options={}] - Retry strategy configuration
* @param {number} [options.maxRetries=3] - Maximum number of retry attempts
* @param {number} [options.baseDelay=1000] - Base delay in milliseconds
* @param {number} [options.maxDelay=30000] - Maximum delay in milliseconds
* @param {number} [options.exponentialBase=2] - Exponential backoff multiplier
* @param {number} [options.jitterFactor=0.1] - Jitter factor (0-1)
* @param {number[]} [options.retryOn] - HTTP status codes to retry on
* @param {boolean} [options.retryOnNetworkError=true] - Retry on network errors
* @param {Function} [options.retryHandler] - Custom retry decision function
* @param {boolean} [options.useStaticDelay=false] - Use static delay instead of exponential
* @param {number} [options.staticRetryDelay] - Static delay when useStaticDelay is true
*
* @example
* const strategy = new RetryStrategy({
* maxRetries: 5,
* baseDelay: 2000,
* exponentialBase: 2,
* jitterFactor: 0.2
* });
*
* @since 2.0.0
*/
constructor(options = {}) {
this.maxRetries = options.maxRetries || 3;
this.baseDelay = options.baseDelay || 1e3;
this.maxDelay = options.maxDelay || 3e4;
this.exponentialBase = options.exponentialBase || 2;
this.jitterFactor = options.jitterFactor || 0.1;
this.retryOn = options.retryOn || [408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524];
this.retryOnNetworkError = options.retryOnNetworkError !== false;
this.retryHandler = options.retryHandler;
this.useStaticDelay = options.useStaticDelay || false;
this.staticRetryDelay = options.staticRetryDelay || this.baseDelay;
}
/**
* Calculate delay for the next retry attempt
*
* @description
* Calculates retry delay using exponential backoff with jitter, respects
* Retry-After headers, and enforces maximum delay limits. Supports both
* dynamic and static delay strategies.
*
* @param {number} attempt - Current attempt number (0-based)
* @description First retry is attempt 0, second is attempt 1, etc.
*
* @param {Object} [error=null] - Error that occurred
* @param {number} [error.retryAfter] - Retry-After value in seconds
* @description When present, takes precedence over calculated delays
*
* @returns {number} Delay in milliseconds
* @description Calculated delay capped at maxDelay
*
* @example
* // Exponential backoff
* strategy.calculateDelay(0); // ~1000ms + jitter
* strategy.calculateDelay(1); // ~2000ms + jitter
* strategy.calculateDelay(2); // ~4000ms + jitter
*
* @example
* // With Retry-After header
* const error = { retryAfter: 30 }; // 30 seconds
* strategy.calculateDelay(0, error); // 30000ms (30 seconds)
*
* @since 2.0.0
*/
calculateDelay(attempt, error = null) {
if (this.useStaticDelay) {
return this.staticRetryDelay;
}
if (error && error.retryAfter) {
return Math.min(error.retryAfter * 1e3, this.maxDelay);
}
const exponentialDelay = this.baseDelay * Math.pow(this.exponentialBase, attempt);
const jitter = exponentialDelay * this.jitterFactor * Math.random();
const totalDelay = exponentialDelay + jitter;
return Math.min(totalDelay, this.maxDelay);
}
/**
* Determine if an error should be retried
*
* @description
* Evaluates whether an error should trigger a retry attempt based on
* error type, status code, attempt count, and custom retry handlers.
* Supports network errors, HTTP status codes, and timeout conditions.
*
* @param {Error} error - Error that occurred
* @param {number} [error.statusCode] - HTTP status code
* @param {string} [error.code] - Error code (e.g., 'ETIMEDOUT', 'ENOTFOUND')
* @param {string} [error.name] - Error name (e.g., 'NetworkError', 'TimeoutError')
*
* @param {number} attempt - Current attempt number (0-based)
* @description Must be less than maxRetries to retry
*
* @returns {boolean} True if the error should be retried
* @description False if max attempts reached or error is not retryable
*
* @example
* strategy.shouldRetry({ statusCode: 500 }, 0); // true (server error)
* strategy.shouldRetry({ statusCode: 404 }, 0); // false (client error)
* strategy.shouldRetry({ code: 'ENOTFOUND' }, 1); // true (network error)
* strategy.shouldRetry({ statusCode: 500 }, 3); // false (max attempts)
*
* @since 2.0.0
*/
shouldRetry(error, attempt) {
if (attempt >= this.maxRetries) {
return false;
}
if (this.retryHandler && typeof this.retryHandler === "function") {
return this.retryHandler(error, attempt);
}
if (this.retryOnNetworkError && this.isNetworkError(error)) {
return true;
}
if (error.statusCode && this.retryOn.includes(error.statusCode)) {
return true;
}
if (error.code === "ETIMEDOUT" || error.name === "TimeoutError") {
return true;
}
return false;
}
/**
* Check if error is a network error
*
* @description
* Identifies network-related errors that should typically be retried.
* Checks error codes, names, and types commonly associated with
* network connectivity issues.
*
* @param {Error} error - Error to check
* @param {string} [error.code] - Error code to check
* @param {string} [error.name] - Error name to check
* @param {string} [error.type] - Error type to check
*
* @returns {boolean} True if the error is network-related
* @description Network errors are typically retryable
*
* @example
* strategy.isNetworkError({ code: 'ENOTFOUND' }); // true
* strategy.isNetworkError({ code: 'ECONNRESET' }); // true
* strategy.isNetworkError({ name: 'NetworkError' }); // true
* strategy.isNetworkError({ statusCode: 400 }); // false
*
* @since 2.0.0
*/
isNetworkError(error) {
const networkErrorCodes = [
"ENOTFOUND",
"ECONNRESET",
"ECONNREFUSED",
"ECONNABORTED",
"EHOSTUNREACH",
"ENETUNREACH",
"EAI_AGAIN"
];
return networkErrorCodes.includes(error.code) || error.name === "NetworkError" || error.type === "NETWORK_ERROR";
}
/**
* Execute a function with retry logic
*
* @description
* Executes a function with automatic retry logic based on the configured
* strategy. Handles delays between attempts, retry decision logic, and
* comprehensive error reporting when all attempts are exhausted.
*
* @param {Function} fn - Async function to execute with retries
* @description Function receives (context, attempt) as parameters
*
* @param {Object} [context={}] - Context object passed to function
* @param {string} [context.url] - URL for error reporting
* @param {string} [context.method] - HTTP method for error reporting
* @param {boolean} [context.verbose] - Enable retry attempt logging
*
* @returns {Promise<any>} Promise resolving to function result
* @description Resolves with function result or rejects with RetryError
*
* @throws {RetryError} When all retry attempts are exhausted
* @throws {Error} When error is not retryable (passes through original error)
*
* @example
* const result = await strategy.execute(async (context, attempt) => {
* console.log(`Attempt ${attempt + 1}`);
* const response = await fetch(context.url);
* if (!response.ok) {
* throw new Error(`HTTP ${response.status}`);
* }
* return response.json();
* }, { url: 'https://api.example.com/data' });
*
* @since 2.0.0
*/
async execute(fn, context = {}) {
let lastError;
let attempt = 0;
while (attempt <= this.maxRetries) {
try {
const result = await fn(context, attempt);
return result;
} catch (error) {
lastError = error;
attempt++;
if (attempt > this.maxRetries) {
throw new RetryError(`All ${this.maxRetries} retry attempts failed`, {
maxRetries: this.maxRetries,
lastError,
url: context.url,
method: context.method
});
}
if (!this.shouldRetry(error, attempt - 1)) {
throw error;
}
const delay = this.calculateDelay(attempt - 1, error);
if (context.verbose) {
console.log(`Retry attempt ${attempt}/${this.maxRetries + 1} after ${delay}ms delay. Error: ${error.message}`);
}
await this.delay(delay);
}
}
throw new RetryError(`All ${this.maxRetries} retry attempts failed`, {
maxRetries: this.maxRetries,
lastError,
url: context.url,
method: context.method
});
}
/**
* Create delay promise
*
* @description
* Creates a promise that resolves after the specified delay.
* Used internally for implementing retry delays.
*
* @param {number} ms - Milliseconds to delay
* @description Must be a non-negative number
*
* @returns {Promise<void>} Promise that resolves after delay
* @description Promise resolves with no value after timeout
*
* @example
* await strategy.delay(1000); // Wait 1 second
* console.log('Delay completed');
*
* @since 2.0.0
*/
delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Parse Retry-After header value
*
* @description
* Parses HTTP Retry-After header values which can be either a number
* of seconds or an HTTP