@xapp/arachne-utils
Version:
131 lines • 5 kB
JavaScript
;
/*! Copyright (c) 2025, XAPP AI */
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DOCUMENT_EXTENSIONS = void 0;
exports.detectDocumentLinks = detectDocumentLinks;
const hasExtension_1 = require("./hasExtension");
/**
* Extension to document type mapping
*/
const EXTENSION_TO_TYPE = {
'.pdf': 'PDF',
'.doc': 'MS_WORD',
'.docx': 'MS_WORD',
'.xls': 'MS_EXCEL',
'.xlsx': 'MS_EXCEL',
'.ppt': 'PPT',
'.pptx': 'PPT'
};
/**
* All supported document extensions
*/
exports.DOCUMENT_EXTENSIONS = Object.keys(EXTENSION_TO_TYPE);
/**
* Extract document type from URL by checking both pathname and query parameters
* @param url - The URL to check
* @returns The document type if found, undefined otherwise
*/
function getDocumentTypeFromUrl(url) {
// First check pathname for extensions
for (const [extension, type] of Object.entries(EXTENSION_TO_TYPE)) {
if ((0, hasExtension_1.hasExtension)(url, extension)) {
return type;
}
}
// Also check query parameters for extensions
// e.g., https://example.com/download?file=report.pdf&token=xyz
try {
const urlObj = new URL(url);
const queryString = urlObj.search.toLowerCase();
for (const [extension, type] of Object.entries(EXTENSION_TO_TYPE)) {
if (queryString.includes(extension)) {
return type;
}
}
}
catch (e) {
// Invalid URL, skip query parameter check
}
return undefined;
}
/**
* Detects document links on a page
*
* This function identifies document links by checking file extensions in both the pathname
* and query parameters of URLs. It detects common document formats including PDF, Word,
* Excel, and PowerPoint files.
*
* **Limitations**:
* - Only detects documents with recognizable file extensions
* - Does not detect documents served via content-type headers without extensions
* - Does not make HEAD requests to verify content types
*
* @param page - The Puppeteer page to scan for document links
* @param allowedTypes - Optional array of document types to filter by (e.g., ['PDF', 'MS_WORD'])
* @returns Promise resolving to an array of detected document links
*
* @example
* ```typescript
* const documentLinks = await detectDocumentLinks(page);
* // Returns all document links
*
* const pdfLinks = await detectDocumentLinks(page, ['PDF']);
* // Returns only PDF links
* ```
*/
function detectDocumentLinks(page, allowedTypes) {
return __awaiter(this, void 0, void 0, function* () {
const baseUrl = page.url();
// Extract all links from the page
const links = yield page.$$eval('a[href]', (anchors, baseUrl) => {
return anchors.map(anchor => {
var _a;
const href = anchor.getAttribute('href');
if (!href)
return null;
try {
// Resolve relative URLs
const absoluteUrl = new URL(href, baseUrl).toString();
const text = ((_a = anchor.textContent) === null || _a === void 0 ? void 0 : _a.trim()) || anchor.getAttribute('title') || '';
return {
url: absoluteUrl,
title: text
};
}
catch (e) {
// Invalid URL, skip it
return null;
}
}).filter(link => link !== null);
}, baseUrl);
// Filter and classify document links
const documentLinks = [];
for (const link of links) {
if (!link)
continue;
// Get document type from URL (checks both pathname and query parameters)
const type = getDocumentTypeFromUrl(link.url);
if (type) {
// If allowedTypes is specified, filter by it
if (!allowedTypes || allowedTypes.includes(type)) {
documentLinks.push({
url: link.url,
type,
title: link.title
});
}
}
}
return documentLinks;
});
}
//# sourceMappingURL=detectDocumentLinks.js.map