@xapp/arachne-utils/lib/downloadDocument.d.ts

/*! Copyright (c) 2025, XAPP AI */
/**
 * Default maximum file size for downloads (50MB)
 */
export declare const DEFAULT_MAX_FILE_SIZE: number;
/**
 * Default download timeout (30 seconds)
 */
export declare const DEFAULT_DOWNLOAD_TIMEOUT = 30000;
/**
 * Options for downloading documents
 */
export interface DownloadDocumentOptions {
    /**
     * Maximum file size in bytes. Downloads larger than this will be rejected.
     * Default: 50MB (52,428,800 bytes)
     */
    maxFileSize?: number;
    /**
     * Download timeout in milliseconds
     * Default: 30000 (30 seconds)
     */
    timeout?: number;
    /**
     * Whether to perform SSRF protection checks
     * Default: true
     */
    ssrfProtection?: boolean;
    /**
     * Optional custom headers to include in the request
     */
    headers?: Record<string, string>;
    /**
     * Whether to validate Content-Type header matches expected document types
     * Default: true
     */
    validateContentType?: boolean;
}
/**
 * Result of a document download operation
 */
export interface DownloadDocumentResult {
    /**
     * The downloaded document as a Buffer
     */
    data: Buffer;
    /**
     * Content-Type header from the response
     */
    contentType?: string;
    /**
     * Content-Length header from the response (file size in bytes)
     */
    contentLength?: number;
    /**
     * The final URL after any redirects
     */
    finalUrl: string;
}
/**
 * Download a document from a URL with safety checks and size limits
 *
 * This function provides:
 * - SSRF protection (blocks private IPs, localhost, metadata endpoints)
 * - File size limits to prevent memory exhaustion
 * - Streaming download with incremental size checking
 * - Timeout handling
 * - Content-Type validation
 *
 * The download uses streaming to protect against memory exhaustion attacks. If a server
 * sends more data than the `maxFileSize` limit, the download is aborted immediately
 * rather than loading the entire file into memory.
 *
 * @param url - The URL of the document to download
 * @param options - Download options
 * @returns Promise resolving to the downloaded document data
 * @throws Error if download fails, exceeds size limit, or violates SSRF protection
 *
 * @example
 * ```typescript
 * try {
 *   const result = await downloadDocument('https://example.com/doc.pdf', {
 *     maxFileSize: 10 * 1024 * 1024, // 10MB
 *     timeout: 15000 // 15 seconds
 *   });
 *
 *   // Save to file
 *   fs.writeFileSync('downloaded.pdf', result.data);
 * } catch (error) {
 *   console.error('Download failed:', error.message);
 * }
 * ```
 */
export declare function downloadDocument(url: string, options?: DownloadDocumentOptions): Promise<DownloadDocumentResult>;