@xapp/arachne-utils
Version:
158 lines • 6.89 kB
JavaScript
;
/*! Copyright (c) 2025, XAPP AI */
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DEFAULT_DOWNLOAD_TIMEOUT = exports.DEFAULT_MAX_FILE_SIZE = void 0;
exports.downloadDocument = downloadDocument;
const ssrfProtection_1 = require("./ssrfProtection");
/**
* Default maximum file size for downloads (50MB)
*/
exports.DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB in bytes
/**
* Default download timeout (30 seconds)
*/
exports.DEFAULT_DOWNLOAD_TIMEOUT = 30000; // 30 seconds
/**
* Expected Content-Type values for document types
*/
const DOCUMENT_CONTENT_TYPES = [
'application/pdf',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.ms-excel',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.ms-powerpoint',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/octet-stream', // Sometimes servers use generic binary type
];
/**
* Download a document from a URL with safety checks and size limits
*
* This function provides:
* - SSRF protection (blocks private IPs, localhost, metadata endpoints)
* - File size limits to prevent memory exhaustion
* - Streaming download with incremental size checking
* - Timeout handling
* - Content-Type validation
*
* The download uses streaming to protect against memory exhaustion attacks. If a server
* sends more data than the `maxFileSize` limit, the download is aborted immediately
* rather than loading the entire file into memory.
*
* @param url - The URL of the document to download
* @param options - Download options
* @returns Promise resolving to the downloaded document data
* @throws Error if download fails, exceeds size limit, or violates SSRF protection
*
* @example
* ```typescript
* try {
* const result = await downloadDocument('https://example.com/doc.pdf', {
* maxFileSize: 10 * 1024 * 1024, // 10MB
* timeout: 15000 // 15 seconds
* });
*
* // Save to file
* fs.writeFileSync('downloaded.pdf', result.data);
* } catch (error) {
* console.error('Download failed:', error.message);
* }
* ```
*/
function downloadDocument(url_1) {
return __awaiter(this, arguments, void 0, function* (url, options = {}) {
var _a;
const { maxFileSize = exports.DEFAULT_MAX_FILE_SIZE, timeout = exports.DEFAULT_DOWNLOAD_TIMEOUT, ssrfProtection = true, headers = {}, validateContentType = true } = options;
// SSRF protection check
if (ssrfProtection && !(0, ssrfProtection_1.isSafeURL)(url)) {
throw new Error(`SSRF protection: URL is not safe to fetch: ${url}`);
}
// Create AbortController for timeout
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const response = yield fetch(url, {
headers: Object.assign({ 'User-Agent': 'Mozilla/5.0 (compatible; Arachne/1.0; +https://github.com/xapp-ai/arachne)' }, headers),
signal: controller.signal,
redirect: 'follow'
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
// Validate Content-Type if requested
const contentType = response.headers.get('content-type');
if (validateContentType && contentType) {
const isValidType = DOCUMENT_CONTENT_TYPES.some(type => contentType.toLowerCase().includes(type.toLowerCase()));
if (!isValidType) {
throw new Error(`Invalid Content-Type: ${contentType}. Expected a document type.`);
}
}
// Check Content-Length before downloading
const contentLengthHeader = response.headers.get('content-length');
if (contentLengthHeader) {
const contentLength = parseInt(contentLengthHeader, 10);
if (contentLength > maxFileSize) {
throw new Error(`File size (${contentLength} bytes) exceeds maximum allowed size (${maxFileSize} bytes)`);
}
}
// Stream the download and check size incrementally to prevent memory exhaustion
const reader = (_a = response.body) === null || _a === void 0 ? void 0 : _a.getReader();
if (!reader) {
throw new Error('Response body is not readable');
}
const chunks = [];
let receivedLength = 0;
try {
while (true) {
const { done, value } = yield reader.read();
if (done)
break;
receivedLength += value.length;
// Check size during download to prevent memory exhaustion
if (receivedLength > maxFileSize) {
reader.cancel();
throw new Error(`File size exceeds maximum allowed size (${maxFileSize} bytes) during download`);
}
chunks.push(value);
}
}
catch (error) {
// Ensure reader is cancelled on any error
try {
yield reader.cancel();
}
catch (_b) {
// Ignore cancellation errors
}
throw error;
}
// Combine chunks into a single buffer
const buffer = Buffer.concat(chunks.map(chunk => Buffer.from(chunk)));
return {
data: buffer,
contentType: contentType || undefined,
contentLength: buffer.length,
finalUrl: response.url
};
}
catch (error) {
if (error.name === 'AbortError') {
throw new Error(`Download timeout after ${timeout}ms`);
}
throw error;
}
finally {
clearTimeout(timeoutId);
}
});
}
//# sourceMappingURL=downloadDocument.js.map