maxun-core
Version:
Core package for Maxun, responsible for data extraction
971 lines • 63.9 kB
JavaScript
/* eslint-disable @typescript-eslint/no-unused-vars */
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
const area = (element) => element.offsetHeight * element.offsetWidth;
function getBiggestElement(selector) {
const elements = Array.from(document.querySelectorAll(selector));
const biggest = elements.reduce((max, elem) => (area(elem) > area(max) ? elem : max), { offsetHeight: 0, offsetWidth: 0 });
return biggest;
}
/**
* Generates structural selector (describing element by its DOM tree location).
*
* **The generated selector is not guaranteed to be unique!** (In fact, this is
* the desired behaviour in here.)
* @param {HTMLElement} element Element being described.
* @returns {string} CSS-compliant selector describing the element's location in the DOM tree.
*/
function GetSelectorStructural(element) {
// Base conditions for the recursive approach.
if (element.tagName === 'BODY') {
return 'BODY';
}
const selector = element.tagName;
if (element.parentElement) {
return `${GetSelectorStructural(element.parentElement)} > ${selector}`;
}
return selector;
}
/**
* Heuristic method to find collections of "interesting" items on the page.
* @returns {Array<HTMLElement>} A collection of interesting DOM nodes
* (online store products, plane tickets, list items... and many more?)
*/
function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, metricType = 'size_deviation') {
const restoreScroll = (() => {
const { scrollX, scrollY } = window;
return () => {
window.scrollTo(scrollX, scrollY);
};
})();
/**
* @typedef {Array<{x: number, y: number}>} Grid
*/
/**
* Returns an array of grid-aligned {x,y} points.
* @param {number} [granularity=0.005] sets the number of generated points
* (the higher the granularity, the more points).
* @returns {Grid} Array of {x, y} objects.
*/
function getGrid(startX = 0, startY = 0, granularity = 0.005) {
const width = window.innerWidth;
const height = window.innerHeight;
const out = [];
for (let x = 0; x < width; x += 1 / granularity) {
for (let y = 0; y < height; y += 1 / granularity) {
out.push({ x: startX + x, y: startY + y });
}
}
return out;
}
let maxSelector = { selector: 'body', metric: 0 };
const updateMaximumWithPoint = (point) => {
const currentElement = document.elementFromPoint(point.x, point.y);
const selector = GetSelectorStructural(currentElement);
const elements = Array.from(document.querySelectorAll(selector))
.filter((element) => area(element) > minArea);
// If the current selector targets less than three elements,
// we consider it not interesting (would be a very underwhelming scraper)
if (elements.length < 3) {
return;
}
let metric = null;
if (metricType === 'total_area') {
metric = elements
.reduce((p, x) => p + area(x), 0);
}
else if (metricType === 'size_deviation') {
// This could use a proper "statistics" approach... but meh, so far so good!
const sizes = elements
.map((element) => area(element));
metric = (1 - (Math.max(...sizes) - Math.min(...sizes)) / Math.max(...sizes));
}
if (metric > maxSelector.metric && elements.length < maxCountPerPage) {
maxSelector = { selector, metric };
}
};
for (let scroll = 0; scroll < scrolls; scroll += 1) {
window.scrollTo(0, scroll * window.innerHeight);
const grid = getGrid();
grid.forEach(updateMaximumWithPoint);
}
restoreScroll();
let out = Array.from(document.querySelectorAll(maxSelector.selector));
const different = (x, i, a) => a.findIndex((e) => e === x) === i;
// as long as we don't merge any two elements by substituing them for their parents,
// we substitute.
while (out.map((x) => x.parentElement).every(different)
&& out.forEach((x) => x.parentElement !== null)) {
out = out.map((x) => { var _a; return (_a = x.parentElement) !== null && _a !== void 0 ? _a : x; });
}
return out;
}
/**
* Returns a "scrape" result from the current page.
* @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed)
*/
// Wrap the entire function in an IIFE (Immediately Invoked Function Expression)
// and attach it to the window object
(function (window) {
/**
* Returns a "scrape" result from the current page.
* @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed)
*/
window.scrape = function (selector = null) {
/**
* **crudeRecords** contains uncurated rundowns of "scrapable" elements
* @type {Array<Object>}
*/
const crudeRecords = (selector
? Array.from(document.querySelectorAll(selector))
: scrapableHeuristics())
.map((record) => (Object.assign(Object.assign({}, Array.from(record.querySelectorAll('img'))
.reduce((p, x, i) => {
let url = null;
if (x.srcset) {
const urls = x.srcset.split(', ');
[url] = urls[urls.length - 1].split(' ');
}
/**
* Contains the largest elements from `srcset` - if `srcset` is not present, contains
* URL from the `src` attribute
*
* If the `src` attribute contains a data url, imgUrl contains `undefined`.
*/
let imgUrl;
if (x.srcset) {
imgUrl = url;
}
else if (x.src.indexOf('data:') === -1) {
imgUrl = x.src;
}
return (Object.assign(Object.assign({}, p), (imgUrl ? { [`img_${i}`]: imgUrl } : {})));
}, {})), record.innerText.split('\n')
.reduce((p, x, i) => (Object.assign(Object.assign({}, p), { [`record_${String(i).padStart(4, '0')}`]: x.trim() })), {}))));
return crudeRecords;
};
/**
* TODO: Simplify.
* Given an object with named lists of elements,
* groups the elements by their distance in the DOM tree.
* @param {Object.<string, {selector: string, tag: string}>} lists The named lists of HTML elements.
* @returns {Array.<Object.<string, string>>}
*/
window.scrapeSchema = function (lists) {
// Utility functions remain the same
function omap(object, f, kf = (x) => x) {
return Object.fromEntries(Object.entries(object)
.map(([k, v]) => [kf(k), f(v)]));
}
function ofilter(object, f) {
return Object.fromEntries(Object.entries(object)
.filter(([k, v]) => f(k, v)));
}
function findAllElements(config) {
var _a;
// Regular DOM query if no special delimiters
if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
return Array.from(document.querySelectorAll(config.selector));
}
if (config.selector.includes(':>>')) {
const parts = config.selector.split(':>>').map(s => s.trim());
let currentElements = [document];
// Traverse through each part of the selector
for (let i = 0; i < parts.length; i++) {
const part = parts[i];
const nextElements = [];
const isLast = i === parts.length - 1;
for (const element of currentElements) {
try {
const doc = element.contentDocument || element || ((_a = element.contentWindow) === null || _a === void 0 ? void 0 : _a.document);
if (!doc)
continue;
if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) {
const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/);
if (nameMatch && nameMatch[1]) {
const frameName = nameMatch[1];
let foundFrames = [];
if (doc.getElementsByName && typeof doc.getElementsByName === 'function') {
foundFrames = Array.from(doc.getElementsByName(frameName))
.filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME');
}
if (foundFrames.length === 0) {
const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`));
foundFrames = framesBySelector;
}
if (isLast) {
nextElements.push(...foundFrames);
}
else {
nextElements.push(...foundFrames);
}
continue;
}
}
const found = Array.from(doc.querySelectorAll(part));
if (isLast) {
nextElements.push(...found);
}
else {
const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME');
nextElements.push(...frames);
}
}
catch (error) {
console.warn('Cannot access iframe/frame content:', error, {
part,
element,
index: i
});
}
}
if (nextElements.length === 0) {
console.warn('No elements found for part:', part, 'at depth:', i);
return [];
}
currentElements = nextElements;
}
return currentElements;
}
// Handle shadow DOM traversal
if (config.selector.includes('>>')) {
const parts = config.selector.split('>>').map(s => s.trim());
let currentElements = [document];
for (const part of parts) {
const nextElements = [];
for (const element of currentElements) {
// Try regular DOM first
const found = Array.from(element.querySelectorAll(part));
// Then check shadow roots
for (const foundEl of found) {
if (foundEl.shadowRoot) {
nextElements.push(foundEl.shadowRoot);
}
else {
nextElements.push(foundEl);
}
}
}
currentElements = nextElements;
}
return currentElements.filter(el => !(el instanceof ShadowRoot));
}
return [];
}
function getElementValue(element, attribute) {
var _a, _b, _c, _d, _e, _f;
if (!element)
return null;
let baseURL;
try {
baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) ||
((_c = element.ownerDocument) === null || _c === void 0 ? void 0 : _c.baseURI) ||
window.location.origin;
}
catch (e) {
baseURL = window.location.origin;
}
switch (attribute) {
case 'href': {
const relativeHref = element.getAttribute('href');
return relativeHref ? new URL(relativeHref, baseURL).href : null;
}
case 'src': {
const relativeSrc = element.getAttribute('src');
return relativeSrc ? new URL(relativeSrc, baseURL).href : null;
}
case 'innerText':
return (_d = element.innerText) === null || _d === void 0 ? void 0 : _d.trim();
case 'textContent':
return (_e = element.textContent) === null || _e === void 0 ? void 0 : _e.trim();
case 'innerHTML':
return element.innerHTML;
case 'outerHTML':
return element.outerHTML;
default:
return element.getAttribute(attribute) || ((_f = element.innerText) === null || _f === void 0 ? void 0 : _f.trim());
}
}
// Rest of the functions remain largely the same
function getSeedKey(listObj) {
const maxLength = Math.max(...Object.values(omap(listObj, (x) => findAllElements(x).length)));
return Object.keys(ofilter(listObj, (_, v) => findAllElements(v).length === maxLength))[0];
}
// Find minimal bounding elements
function getMBEs(elements) {
return elements.map((element) => {
let candidate = element;
const isUniqueChild = (e) => elements
.filter((elem) => {
var _a;
// Handle both iframe and shadow DOM boundaries
const sameContext = elem.getRootNode() === e.getRootNode() &&
elem.ownerDocument === e.ownerDocument;
return sameContext && ((_a = e.parentNode) === null || _a === void 0 ? void 0 : _a.contains(elem));
})
.length === 1;
while (candidate && isUniqueChild(candidate)) {
candidate = candidate.parentNode;
}
return candidate;
});
}
const seedName = getSeedKey(lists);
const seedElements = findAllElements(lists[seedName]);
const MBEs = getMBEs(seedElements);
const mbeResults = MBEs.map((mbe) => omap(lists, (config) => {
const elem = findAllElements(config)
.find((elem) => mbe.contains(elem));
return elem ? getElementValue(elem, config.attribute) : undefined;
}, (key) => key)) || [];
// If MBE approach didn't find all elements, try independent scraping
if (mbeResults.some(result => Object.values(result).some(v => v === undefined))) {
// Fall back to independent scraping
const results = [];
const foundElements = new Map();
// Find all elements for each selector
Object.entries(lists).forEach(([key, config]) => {
const elements = findAllElements(config);
foundElements.set(key, elements);
});
// Create result objects for each found element
foundElements.forEach((elements, key) => {
elements.forEach((element, index) => {
if (!results[index]) {
results[index] = {};
}
results[index][key] = getElementValue(element, lists[key].attribute);
});
});
return results.filter(result => Object.keys(result).length > 0);
}
return mbeResults;
};
/**
* Scrapes multiple lists of similar items based on a template item.
* @param {Object} config - Configuration object
* @param {string} config.listSelector - Selector for the list container(s)
* @param {Object.<string, {selector: string, attribute?: string}>} config.fields - Fields to scrape
* @param {number} [config.limit] - Maximum number of items to scrape per list (optional)
* @param {boolean} [config.flexible=false] - Whether to use flexible matching for field selectors
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
*/
window.scrapeList = function ({ listSelector, fields, limit = 10 }) {
return __awaiter(this, void 0, void 0, function* () {
// XPath evaluation functions
const queryInsideContext = (context, part) => {
try {
const { tagName, conditions } = parseXPathPart(part);
const candidateElements = Array.from(context.querySelectorAll(tagName));
if (candidateElements.length === 0) {
return [];
}
const matchingElements = candidateElements.filter((el) => {
return elementMatchesConditions(el, conditions);
});
return matchingElements;
}
catch (err) {
console.error("Error in queryInsideContext:", err);
return [];
}
};
// Helper function to parse XPath part
const parseXPathPart = (part) => {
const tagMatch = part.match(/^([a-zA-Z0-9-]+)/);
const tagName = tagMatch ? tagMatch[1] : "*";
const conditionMatches = part.match(/\[([^\]]+)\]/g);
const conditions = conditionMatches
? conditionMatches.map((c) => c.slice(1, -1))
: [];
return { tagName, conditions };
};
// Helper function to check if element matches all conditions
const elementMatchesConditions = (element, conditions) => {
for (const condition of conditions) {
if (!elementMatchesCondition(element, condition)) {
return false;
}
}
return true;
};
// Helper function to check if element matches a single condition
const elementMatchesCondition = (element, condition) => {
var _a, _b;
condition = condition.trim();
if (/^\d+$/.test(condition)) {
return true;
}
// Handle @attribute="value"
const attrMatch = condition.match(/^@([^=]+)=["']([^"']+)["']$/);
if (attrMatch) {
const [, attr, value] = attrMatch;
const elementValue = element.getAttribute(attr);
return elementValue === value;
}
// Handle contains(@class, 'value')
const classContainsMatch = condition.match(/^contains\(@class,\s*["']([^"']+)["']\)$/);
if (classContainsMatch) {
const className = classContainsMatch[1];
return element.classList.contains(className);
}
// Handle contains(@attribute, 'value')
const attrContainsMatch = condition.match(/^contains\(@([^,]+),\s*["']([^"']+)["']\)$/);
if (attrContainsMatch) {
const [, attr, value] = attrContainsMatch;
const elementValue = element.getAttribute(attr) || "";
return elementValue.includes(value);
}
// Handle text()="value"
const textMatch = condition.match(/^text\(\)=["']([^"']+)["']$/);
if (textMatch) {
const expectedText = textMatch[1];
const elementText = ((_a = element.textContent) === null || _a === void 0 ? void 0 : _a.trim()) || "";
return elementText === expectedText;
}
// Handle contains(text(), 'value')
const textContainsMatch = condition.match(/^contains\(text\(\),\s*["']([^"']+)["']\)$/);
if (textContainsMatch) {
const expectedText = textContainsMatch[1];
const elementText = ((_b = element.textContent) === null || _b === void 0 ? void 0 : _b.trim()) || "";
return elementText.includes(expectedText);
}
// Handle count(*)=0 (element has no children)
if (condition === "count(*)=0") {
return element.children.length === 0;
}
// Handle other count conditions
const countMatch = condition.match(/^count\(\*\)=(\d+)$/);
if (countMatch) {
const expectedCount = parseInt(countMatch[1]);
return element.children.length === expectedCount;
}
return true;
};
const evaluateXPath = (document, xpath, isShadow = false) => {
try {
if (!document || !xpath) {
console.warn('Invalid document or xpath provided to evaluateXPath');
return null;
}
const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
if (!isShadow) {
if (result === null) {
return null;
}
return result;
}
let cleanPath = xpath;
let isIndexed = false;
const indexedMatch = xpath.match(/^\((.*?)\)\[(\d+)\](.*)$/);
if (indexedMatch) {
cleanPath = indexedMatch[1] + indexedMatch[3];
isIndexed = true;
}
const pathParts = cleanPath
.replace(/^\/\//, "")
.split("/")
.map((p) => p.trim())
.filter((p) => p.length > 0);
let currentContexts = [document];
for (let i = 0; i < pathParts.length; i++) {
const part = pathParts[i];
const nextContexts = [];
for (const ctx of currentContexts) {
const positionalMatch = part.match(/^([^[]+)\[(\d+)\]$/);
let partWithoutPosition = part;
let requestedPosition = null;
if (positionalMatch) {
partWithoutPosition = positionalMatch[1];
requestedPosition = parseInt(positionalMatch[2]);
}
const matched = queryInsideContext(ctx, partWithoutPosition);
let elementsToAdd = matched;
if (requestedPosition !== null) {
const index = requestedPosition - 1; // XPath is 1-based, arrays are 0-based
if (index >= 0 && index < matched.length) {
elementsToAdd = [matched[index]];
}
else {
console.warn(`Position ${requestedPosition} out of range (${matched.length} elements found)`);
elementsToAdd = [];
}
}
elementsToAdd.forEach((el) => {
nextContexts.push(el);
if (el.shadowRoot) {
nextContexts.push(el.shadowRoot);
}
});
}
if (nextContexts.length === 0) {
return null;
}
currentContexts = nextContexts;
}
if (currentContexts.length > 0) {
if (isIndexed && indexedMatch) {
const requestedIndex = parseInt(indexedMatch[2]) - 1;
if (requestedIndex >= 0 && requestedIndex < currentContexts.length) {
return currentContexts[requestedIndex];
}
else {
console.warn(`Requested index ${requestedIndex + 1} out of range (${currentContexts.length} elements found)`);
return null;
}
}
return currentContexts[0];
}
return null;
}
catch (err) {
console.error("Critical XPath failure:", xpath, err);
// Return null instead of throwing to prevent crashes
return null;
}
};
const evaluateXPathAll = (rootElement, xpath) => {
try {
const ownerDoc = rootElement.nodeType === Node.DOCUMENT_NODE
? rootElement
: rootElement.ownerDocument;
if (!ownerDoc)
return [];
const result = ownerDoc.evaluate(xpath, rootElement, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
const elements = [];
for (let i = 0; i < result.snapshotLength; i++) {
const node = result.snapshotItem(i);
if (node && node.nodeType === Node.ELEMENT_NODE) {
elements.push(node);
}
}
return elements;
}
catch (error) {
console.warn("XPath evaluation failed:", xpath, error);
return [];
}
};
// Helper function to detect selector type
const isXPathSelector = (selector) => {
return (selector.startsWith("//") ||
selector.startsWith("/") ||
selector.startsWith("./"));
};
// Enhanced query function to handle iframe, frame, shadow DOM, CSS selectors, and XPath
const queryElement = (rootElement, selector) => {
var _a;
if (!selector.includes(">>") && !selector.includes(":>>")) {
// Check if it's an XPath selector
if (isXPathSelector(selector)) {
return evaluateXPath(rootElement, selector);
}
else {
return rootElement.querySelector(selector);
}
}
const parts = selector.split(/(?:>>|:>>)/).map((part) => part.trim());
let currentElement = rootElement;
for (let i = 0; i < parts.length; i++) {
if (!currentElement)
return null;
// Handle iframe and frame traversal with enhanced safety
if (currentElement.tagName === "IFRAME" ||
currentElement.tagName === "FRAME") {
try {
// Check if frame is accessible
if (!currentElement.contentDocument && !currentElement.contentWindow) {
console.warn('Frame is not accessible (cross-origin or unloaded)');
return null;
}
const frameDoc = currentElement.contentDocument ||
((_a = currentElement.contentWindow) === null || _a === void 0 ? void 0 : _a.document);
if (!frameDoc) {
console.warn('Frame document is not available');
return null;
}
if (isXPathSelector(parts[i])) {
currentElement = evaluateXPath(frameDoc, parts[i]);
}
else {
currentElement = frameDoc.querySelector(parts[i]);
}
continue;
}
catch (e) {
console.warn(`Cannot access ${currentElement.tagName.toLowerCase()} content:`, e);
return null;
}
}
let nextElement = null;
// Try regular DOM first
if ("querySelector" in currentElement) {
if (isXPathSelector(parts[i])) {
nextElement = evaluateXPath(currentElement, parts[i]);
}
else {
nextElement = currentElement.querySelector(parts[i]);
}
}
// Try shadow DOM if not found
if (!nextElement &&
"shadowRoot" in currentElement &&
currentElement.shadowRoot) {
if (isXPathSelector(parts[i])) {
nextElement = evaluateXPath(currentElement.shadowRoot, parts[i]);
}
else {
nextElement = currentElement.shadowRoot.querySelector(parts[i]);
}
}
// Check children's shadow roots if still not found
if (!nextElement && "children" in currentElement) {
const children = Array.from(currentElement.children || []);
for (const child of children) {
if (child.shadowRoot) {
if (isXPathSelector(parts[i])) {
nextElement = evaluateXPath(child.shadowRoot, parts[i]);
}
else {
nextElement = child.shadowRoot.querySelector(parts[i]);
}
if (nextElement)
break;
}
}
}
currentElement = nextElement;
}
return currentElement;
};
// Enhanced query all function for both contexts
const queryElementAll = (rootElement, selector) => {
if (!selector.includes(">>") && !selector.includes(":>>")) {
if (isXPathSelector(selector)) {
return evaluateXPathAll(rootElement, selector);
}
else {
return Array.from(rootElement.querySelectorAll(selector));
}
}
const parts = selector.split(/(?:>>|:>>)/).map((part) => part.trim());
let currentElements = [rootElement];
for (const part of parts) {
const nextElements = [];
for (const element of currentElements) {
// Handle iframe and frame traversal
if (element.tagName === "IFRAME" || element.tagName === "FRAME") {
try {
const frameDoc = element.contentDocument || element.contentWindow.document;
if (frameDoc) {
if (isXPathSelector(part)) {
nextElements.push(...evaluateXPathAll(frameDoc, part));
}
else {
nextElements.push(...Array.from(frameDoc.querySelectorAll(part)));
}
}
}
catch (e) {
console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e);
continue;
}
}
else {
// Regular DOM elements
if (element.querySelectorAll) {
if (isXPathSelector(part)) {
nextElements.push(...evaluateXPathAll(element, part));
}
else {
nextElements.push(...Array.from(element.querySelectorAll(part)));
}
}
// Shadow DOM elements
if (element.shadowRoot) {
if (isXPathSelector(part)) {
nextElements.push(...evaluateXPathAll(element.shadowRoot, part));
}
else {
nextElements.push(...Array.from(element.shadowRoot.querySelectorAll(part)));
}
}
// Check children's shadow roots
const children = Array.from(element.children || []);
for (const child of children) {
if (child.shadowRoot) {
if (isXPathSelector(part)) {
nextElements.push(...evaluateXPathAll(child.shadowRoot, part));
}
else {
nextElements.push(...Array.from(child.shadowRoot.querySelectorAll(part)));
}
}
}
}
}
currentElements = nextElements;
}
return currentElements;
};
// Enhanced value extraction with context awareness
const extractValue = (element, attribute) => {
var _a, _b, _c, _d, _e;
if (!element)
return null;
// Get context-aware base URL
const baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) || window.location.origin;
// Check shadow root first
if (element.shadowRoot) {
const shadowContent = element.shadowRoot.textContent;
if (shadowContent === null || shadowContent === void 0 ? void 0 : shadowContent.trim()) {
return shadowContent.trim();
}
}
if (attribute === "innerText") {
// First try standard innerText/textContent
let textContent = ((_c = element.innerText) === null || _c === void 0 ? void 0 : _c.trim()) || ((_d = element.textContent) === null || _d === void 0 ? void 0 : _d.trim());
// If empty, check for common data attributes that might contain the text
if (!textContent) {
const dataAttributes = [
"data-600",
"data-text",
"data-label",
"data-value",
"data-content",
];
for (const attr of dataAttributes) {
const dataValue = element.getAttribute(attr);
if (dataValue && dataValue.trim()) {
textContent = dataValue.trim();
break;
}
}
}
return textContent || null;
}
else if (attribute === "innerHTML") {
return ((_e = element.innerHTML) === null || _e === void 0 ? void 0 : _e.trim()) || null;
}
else if (attribute === "src" || attribute === "href") {
if (attribute === "href" && element.tagName !== "A") {
const parentElement = element.parentElement;
if (parentElement && parentElement.tagName === "A") {
const parentHref = parentElement.getAttribute("href");
if (parentHref) {
try {
return new URL(parentHref, baseURL).href;
}
catch (e) {
return parentHref;
}
}
}
}
const attrValue = element.getAttribute(attribute);
const dataAttr = attrValue || element.getAttribute("data-" + attribute);
if (!dataAttr || dataAttr.trim() === "") {
if (attribute === "src") {
const style = window.getComputedStyle(element);
const bgImage = style.backgroundImage;
if (bgImage && bgImage !== "none") {
const matches = bgImage.match(/url\(['"]?([^'")]+)['"]?\)/);
return matches ? new URL(matches[1], baseURL).href : null;
}
}
return null;
}
try {
return new URL(dataAttr, baseURL).href;
}
catch (e) {
console.warn("Error creating URL from", dataAttr, e);
return dataAttr;
}
}
return element.getAttribute(attribute);
};
// Enhanced table ancestor finding with context support
const findTableAncestor = (element) => {
let currentElement = element;
const MAX_DEPTH = 5;
let depth = 0;
while (currentElement && depth < MAX_DEPTH) {
// Handle shadow DOM
if (currentElement.getRootNode() instanceof ShadowRoot) {
currentElement = currentElement.getRootNode().host;
continue;
}
if (currentElement.tagName === "TD") {
return { type: "TD", element: currentElement };
}
else if (currentElement.tagName === "TR") {
return { type: "TR", element: currentElement };
}
// Handle iframe and frame crossing
if (currentElement.tagName === "IFRAME" ||
currentElement.tagName === "FRAME") {
try {
currentElement = currentElement.contentDocument.body;
}
catch (e) {
return null;
}
}
else {
currentElement = currentElement.parentElement;
}
depth++;
}
return null;
};
// Helper function to get cell index
const getCellIndex = (td) => {
if (td.getRootNode() instanceof ShadowRoot) {
const shadowRoot = td.getRootNode();
const allCells = Array.from(shadowRoot.querySelectorAll("td"));
return allCells.indexOf(td);
}
let index = 0;
let sibling = td;
while ((sibling = sibling.previousElementSibling)) {
index++;
}
return index;
};
// Helper function to check for TH elements
const hasThElement = (row, tableFields) => {
for (const [_, { selector }] of Object.entries(tableFields)) {
const element = queryElement(row, selector);
if (element) {
let current = element;
while (current && current !== row) {
if (current.getRootNode() instanceof ShadowRoot) {
current = current.getRootNode().host;
continue;
}
if (current.tagName === "TH")
return true;
if (current.tagName === "IFRAME" || current.tagName === "FRAME") {
try {
current = current.contentDocument.body;
}
catch (e) {
break;
}
}
else {
current = current.parentElement;
}
}
}
}
return false;
};
// Helper function to filter rows
const filterRowsBasedOnTag = (rows, tableFields) => {
for (const row of rows) {
if (hasThElement(row, tableFields)) {
return rows;
}
}
return rows.filter((row) => {
const directTH = row.getElementsByTagName("TH").length === 0;
const shadowTH = row.shadowRoot
? row.shadowRoot.querySelector("th") === null
: true;
return directTH && shadowTH;
});
};
// Class similarity comparison functions
const calculateClassSimilarity = (classList1, classList2) => {
const set1 = new Set(classList1);
const set2 = new Set(classList2);
const intersection = new Set([...set1].filter((x) => set2.has(x)));
const union = new Set([...set1, ...set2]);
return intersection.size / union.size;
};
// Enhanced similar elements finding with context support
const findSimilarElements = (baseElement, similarityThreshold = 0.7) => {
const baseClasses = Array.from(baseElement.classList);
if (baseClasses.length === 0)
return [];
const allElements = [];
// Get elements from main document
allElements.push(...document.getElementsByTagName(baseElement.tagName));
// Get elements from shadow DOM
if (baseElement.getRootNode() instanceof ShadowRoot) {
const shadowHost = baseElement.getRootNode().host;
allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
}
// Get elements from iframes and frames
const frames = [
...Array.from(document.getElementsByTagName("iframe")),
...Array.from(document.getElementsByTagName("frame")),
];
for (const frame of frames) {
try {
const frameDoc = frame.contentDocument || frame.contentWindow.document;
allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName));
}
catch (e) {
console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e);
}
}
return allElements.filter((element) => {
if (element === baseElement)
return false;
const similarity = calculateClassSimilarity(baseClasses, Array.from(element.classList));
return similarity >= similarityThreshold;
});
};
const tryFallbackSelector = (rootElement, originalSelector) => {
let element = queryElement(rootElement, originalSelector);
if (!element && originalSelector.includes("nth-child")) {
const match = originalSelector.match(/nth-child\((\d+)\)/);
if (match) {
const position = parseInt(match[1], 10);
for (let i = position - 1; i >= 1; i--) {
const fallbackSelector = originalSelector.replace(/nth-child\(\d+\)/, `nth-child(${i})`);
element = queryElement(rootElement, fallbackSelector);
if (element)
break;
}
if (!element) {
const baseSelector = originalSelector.replace(/\:nth-child\(\d+\)/, "");
element = queryElement(rootElement, baseSelector);
}
}
}
return element;
};
// Create indexed XPath for specific container instance
const createIndexedXPath = (childSelector, listSelector, containerIndex) => {
// Check if the child selector contains the list selector pattern
if (childSelector.includes(listSelector.replace("//", ""))) {
// Replace the list selector part with indexed version
const listPattern = listSelector.replace("//", "");
const indexedListSelector = `(${listSelector})[${containerIndex}]`;
const indexedSelector = childSelector.replace(`//${listPattern}`, indexedListSelector);
return indexedSelector;
}
else {
// If pattern doesn't match, create a more generic indexed selector
return `(${listSelector})[${containerIndex}]${childSelector.replace("//", "/")}`;
}
};
// Main scraping logic with unified support for both CSS and XPath
console.log("🚀 Starting unified list data extraction");
console.log("List Selector:", listSelector);
console.log("Fields:", fields);
let containers = queryElementAll(document, listSelector);
containers = Array.from(containers);
if (containers.length === 0) {
console.warn("❌ No containers found for listSelector:", listS