@microfox/puppeteer-sls
Version:
Puppeteer SLS - Run puppeteer on a serverless function (AWS Lambda, etc.)
974 lines (963 loc) • 35.7 kB
JavaScript
;
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var index_exports = {};
__export(index_exports, {
PuppeteerSLS: () => PuppeteerSLS,
extractBackgroundImages: () => extractBackgroundImages,
extractHTML: () => extractHTML,
extractImages: () => extractImages,
extractImagesFromURL: () => extractImagesFromURL,
extractLinks: () => extractLinks,
extractLinksFromUrl: () => extractLinksFromUrl,
extractToMarkdown: () => extractToMarkdown,
extractVideos: () => extractVideos,
extractVideosFromURL: () => extractVideosFromURL,
extractWebpage: () => extractWebpage,
openPage: () => openPage,
puppeteerLaunchProps: () => puppeteerLaunchProps,
takeSnapShot: () => takeSnapShot
});
module.exports = __toCommonJS(index_exports);
// src/functions/openPage.ts
var import_puppeteer_extra = __toESM(require("puppeteer-extra"));
var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"));
// src/spartacuz.ts
var import_chromium = __toESM(require("@sparticuz/chromium"));
var fs = __toESM(require("fs"));
var isLocal = process.env.IS_OFFLINE || process.env.SERVERLESS_OFFLINE;
import_chromium.default.setGraphicsMode = false;
var puppeteerLaunchProps = async (defaultViewport, _isLocal, headless) => {
let executablePath;
console.log("isLocal", isLocal, process.env.CHROME_EXECUTABLE_PATH);
const CHROME_PATHS = {
darwin: [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
"~/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"~/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"~/Applications/Chromium.app/Contents/MacOS/Chromium"
],
linux: [
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/google-chrome-beta",
"/usr/bin/google-chrome-unstable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
"/usr/bin/microsoft-edge",
"/usr/bin/microsoft-edge-stable",
"/usr/bin/microsoft-edge-beta",
"/usr/bin/microsoft-edge-dev",
"/snap/bin/chromium",
"/usr/local/bin/chrome",
"/usr/local/bin/chromium",
"/opt/google/chrome/chrome",
"/opt/google/chrome-beta/chrome",
"/opt/google/chrome-unstable/chrome",
"/opt/chromium/chrome-linux/chrome",
"/opt/microsoft/msedge/msedge",
"/opt/brave.com/brave/brave",
// Flatpak paths
"~/.local/share/flatpak/exports/bin/org.chromium.Chromium",
"/var/lib/flatpak/exports/bin/org.chromium.Chromium"
],
win32: [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
"C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
"C:\\Program Files\\Brave Software\\Brave-Browser\\Application\\brave.exe",
"C:\\Program Files (x86)\\Brave Software\\Brave-Browser\\Application\\brave.exe",
// User-specific locations with AppData
`${process.env.LOCALAPPDATA}\\Google\\Chrome\\Application\\chrome.exe`,
`${process.env.LOCALAPPDATA}\\Google\\Chrome SxS\\Application\\chrome.exe`,
`${process.env.LOCALAPPDATA}\\Chromium\\Application\\chrome.exe`,
`${process.env.LOCALAPPDATA}\\Microsoft\\Edge\\Application\\msedge.exe`,
`${process.env.LOCALAPPDATA}\\BraveSoftware\\Brave-Browser\\Application\\brave.exe`,
// Windows 10/11 may have Edge installed in Program Files
"C:\\Program Files (x86)\\Microsoft\\Edge Dev\\Application\\msedge.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge Beta\\Application\\msedge.exe",
// Google Chrome default installation path
`${process.env.ProgramFiles}\\Google\\Chrome\\Application\\chrome.exe`,
`${process.env.ProgramFiles}\\Google\\Chrome Dev\\Application\\chrome.exe`,
`${process.env.ProgramFiles}\\Google\\Chrome Beta\\Application\\chrome.exe`,
`${process.env.ProgramFiles}\\Google\\Chrome SxS\\Application\\chrome.exe`,
// Chromium
`${process.env.ProgramFiles}\\Chromium\\Application\\chrome.exe`
]
};
if (process.env.CHROME_EXECUTABLE_PATH) {
executablePath = process.env.CHROME_EXECUTABLE_PATH;
} else if (isLocal || _isLocal) {
const platform = process.platform;
const paths = CHROME_PATHS[platform];
if (paths) {
for (const p of paths) {
if (fs.existsSync(p)) {
executablePath = p;
break;
}
}
}
if (!executablePath) {
console.error(
"Could not find Chrome in default locations. Please install Chrome or specify the path manually via the CHROME_EXECUTABLE_PATH environment variable."
);
throw new Error("Chrome not found");
}
} else {
executablePath = await import_chromium.default.executablePath();
}
const finalHeadless = isLocal || _isLocal ? headless != null ? headless : false : true;
const args = finalHeadless ? import_chromium.default.args : import_chromium.default.args.filter((arg) => !arg.startsWith("--headless"));
console.log("executablePath", executablePath, finalHeadless);
return {
args: [
...args,
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu"
],
defaultViewport: {
width: (defaultViewport == null ? void 0 : defaultViewport.width) || 1920,
height: (defaultViewport == null ? void 0 : defaultViewport.height) || 1080
},
executablePath,
headless: finalHeadless
};
};
// src/functions/openPage.ts
import_puppeteer_extra.default.use((0, import_puppeteer_extra_plugin_stealth.default)());
async function openPage({
url,
defaultViewport,
headless,
isLocal: isLocal2,
waitUntil
}) {
console.log(
`Opening page with options: ${JSON.stringify({
url,
defaultViewport,
headless,
isLocal: isLocal2
})}`
);
try {
const launchProps = await puppeteerLaunchProps(
defaultViewport,
isLocal2 != null ? isLocal2 : false,
headless
);
console.log("Got puppeteer launch properties.");
const browser = await import_puppeteer_extra.default.launch(
!headless ? {
...launchProps,
slowMo: 100
} : launchProps
);
console.log("Browser launched.");
const page = await browser.newPage();
console.log("New page created.");
const pageLoad = await page.goto(url, {
waitUntil: waitUntil != null ? waitUntil : "networkidle2"
});
console.log(`Navigated to URL: ${pageLoad == null ? void 0 : pageLoad.url()}`);
return { browser, page };
} catch (error) {
console.error("Error opening page:", error);
throw error;
}
}
// src/functions/takeSnapShot.ts
var import_s3_space = require("@microfox/s3-space");
async function takeSnapShot(options) {
const { browser, page } = await openPage(options);
const screenshotBuffer = await page.screenshot({
quality: options.quality,
encoding: options.encoding || "binary",
type: options.quality ? "jpeg" : "png"
});
await browser.close();
const s3 = new import_s3_space.S3Space(options.s3Config);
const fileInfo = {
fileName: options.fileName,
fileType: options.quality ? "image/jpeg" : "image/png",
fileBuffer: screenshotBuffer
};
await s3.uploadFile({
fileInfoWithFileBuffer: fileInfo
});
return s3.getPublicFileUrl({
file: fileInfo
});
}
// src/functions/extractWebpage.ts
var import_turndown = __toESM(require("turndown"));
async function extractToMarkdown(page) {
const html = await page.evaluate(() => document.body.innerHTML);
const turndownService = new import_turndown.default();
const markdown = turndownService.turndown(html);
return markdown;
}
async function extractHTML(page) {
const content = await page.evaluate(() => document.documentElement.outerHTML);
return content;
}
async function extractWebpage(options) {
const { browser, page } = await openPage(options);
const content = await page.evaluate(() => document.body.innerText);
const markdown = options.toMarkdown ? await extractToMarkdown(page) : void 0;
const html = options.toHTML ? await extractHTML(page) : void 0;
return {
content,
browser,
markdown,
html,
page
};
}
// src/functions/extractLinks.ts
async function extractLinks(page) {
const links = await page.evaluate(() => {
const imageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".svg"];
const videoExtensions = [".mp4", ".mov", ".avi"];
const fileExtensions = [
".pdf",
".doc",
".docx",
".xls",
".xlsx",
".zip",
".rar"
];
return Array.from(
document.querySelectorAll("a[href]")
).map((anchor) => {
const hrefAttr = anchor.getAttribute("href") || "";
const href = anchor.href;
const displayName = anchor.innerText.trim();
const inPageLink = hrefAttr.startsWith("#");
let externalLink = false;
let isWebLink = false;
let url;
try {
url = new URL(href);
isWebLink = true;
externalLink = url.hostname !== window.location.hostname;
} catch (e) {
isWebLink = false;
externalLink = false;
}
const internalLink = isWebLink && !externalLink && !inPageLink;
const linkResult = {
href,
displayName,
type: "webpage",
inPageLink,
externalLink,
internalLink
};
if (url) {
const pathname = url.pathname;
const extMatch = pathname.match(/\.([^/.]+)$/);
if (extMatch) {
const ext = `.${extMatch[1].toLowerCase()}`;
if (imageExtensions.includes(ext)) {
linkResult.type = "statix";
linkResult.staticType = "image";
linkResult.fileExtension = ext.substring(1);
} else if (videoExtensions.includes(ext)) {
linkResult.type = "statix";
linkResult.staticType = "video";
linkResult.fileExtension = ext.substring(1);
} else if (fileExtensions.includes(ext)) {
linkResult.type = "statix";
linkResult.staticType = "file";
linkResult.fileExtension = ext.substring(1);
}
}
}
return linkResult;
});
});
return links;
}
async function extractLinksFromUrl(options) {
const { browser, page } = await openPage(options);
const links = await extractLinks(page);
await browser.close();
return links;
}
// src/functions/colorExtract.ts
function extractPaletteFromImageData(imageData, options = { enableLogs: false }) {
var _a, _b, _c, _d, _e, _f;
const {
deltaEThreshold = 0.5,
paletteSize = 12,
enableLogs = false
} = options;
const logs = [];
const log = (message) => {
if (enableLogs) {
logs.push(message);
}
};
const hexToRgb = (hex) => {
const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex);
return result ? {
r: parseInt(result[1], 16),
g: parseInt(result[2], 16),
b: parseInt(result[3], 16)
} : null;
};
const getLuminance = (rgb) => {
const a = [rgb.r, rgb.g, rgb.b].map((v) => {
v /= 255;
return v <= 0.03928 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4);
});
return a[0] * 0.2126 + a[1] * 0.7152 + a[2] * 0.0722;
};
const getContrastRatio = (rgb1, rgb2) => {
const lum1 = getLuminance(rgb1);
const lum2 = getLuminance(rgb2);
const brightest = Math.max(lum1, lum2);
const darkest = Math.min(lum1, lum2);
return (brightest + 0.05) / (darkest + 0.05);
};
const rgbToHsl = (rgb) => {
const r = rgb.r / 255;
const g = rgb.g / 255;
const b = rgb.b / 255;
const cmin = Math.min(r, g, b);
const cmax = Math.max(r, g, b);
const delta = cmax - cmin;
let h = 0;
let s = 0;
let l = 0;
if (delta == 0) h = 0;
else if (cmax == r) h = (g - b) / delta % 6;
else if (cmax == g) h = (b - r) / delta + 2;
else h = (r - g) / delta + 4;
h = Math.round(h * 60);
if (h < 0) h += 360;
l = (cmax + cmin) / 2;
s = delta == 0 ? 0 : delta / (1 - Math.abs(2 * l - 1));
return { h, s, l };
};
const rgbToLab = (rgb) => {
let r = rgb.r / 255, g = rgb.g / 255, b = rgb.b / 255;
r = r > 0.04045 ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
g = g > 0.04045 ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
b = b > 0.04045 ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;
let x = r * 0.4124 + g * 0.3576 + b * 0.1805;
let y = r * 0.2126 + g * 0.7152 + b * 0.0722;
let z = r * 0.0193 + g * 0.1192 + b * 0.9505;
x /= 95.047;
y /= 100;
z /= 108.883;
x = x > 8856e-6 ? Math.pow(x, 1 / 3) : 7.787 * x + 16 / 116;
y = y > 8856e-6 ? Math.pow(y, 1 / 3) : 7.787 * y + 16 / 116;
z = z > 8856e-6 ? Math.pow(z, 1 / 3) : 7.787 * z + 16 / 116;
return { l: 116 * y - 16, a: 500 * (x - y), b: 200 * (y - z) };
};
const deltaE = (labA, labB) => {
const deltaL = labA.l - labB.l;
const deltaA = labA.a - labB.a;
const deltaB = labA.b - labB.b;
return Math.sqrt(deltaL * deltaL + deltaA * deltaA + deltaB * deltaB);
};
const rgbToHex = (pixel) => {
const componentToHex = (c) => {
const hex = c.toString(16);
return hex.length == 1 ? "0" + hex : hex;
};
return ("#" + componentToHex(pixel.r) + componentToHex(pixel.g) + componentToHex(pixel.b)).toUpperCase();
};
const buildRgb = (imageData2, sampleRate = 5) => {
const rgbValues = [];
for (let i = 0; i < imageData2.length; i += sampleRate * 4) {
rgbValues.push({
r: imageData2[i],
// Red channel
g: imageData2[i + 1],
// Green channel
b: imageData2[i + 2]
// Blue channel
// Note: Alpha channel (i + 3) is ignored
});
}
return rgbValues;
};
const findBiggestColorRange = (rgbValues) => {
let rMin = Number.MAX_VALUE, gMin = Number.MAX_VALUE, bMin = Number.MAX_VALUE;
let rMax = Number.MIN_VALUE, gMax = Number.MIN_VALUE, bMax = Number.MIN_VALUE;
rgbValues.forEach((pixel) => {
rMin = Math.min(rMin, pixel.r);
gMin = Math.min(gMin, pixel.g);
bMin = Math.min(bMin, pixel.b);
rMax = Math.max(rMax, pixel.r);
gMax = Math.max(gMax, pixel.g);
bMax = Math.max(bMax, pixel.b);
});
const rRange = rMax - rMin;
const gRange = gMax - gMin;
const bRange = bMax - bMin;
const biggestRange = Math.max(rRange, gRange, bRange);
if (biggestRange === rRange) return "r";
if (biggestRange === gRange) return "g";
return "b";
};
const quantization = (rgbValues, depth) => {
const MAX_DEPTH = 5;
if (depth === MAX_DEPTH || rgbValues.length === 0) {
const color = rgbValues.reduce(
(prev, curr) => {
prev.r += curr.r;
prev.g += curr.g;
prev.b += curr.b;
return prev;
},
{ r: 0, g: 0, b: 0 }
);
color.r = Math.round(color.r / rgbValues.length);
color.g = Math.round(color.g / rgbValues.length);
color.b = Math.round(color.b / rgbValues.length);
return [color];
}
const componentToSortBy = findBiggestColorRange(rgbValues);
rgbValues.sort((p1, p2) => p1[componentToSortBy] - p2[componentToSortBy]);
const mid = Math.floor(rgbValues.length / 2);
return [
...quantization(rgbValues.slice(0, mid), depth + 1),
...quantization(rgbValues.slice(mid), depth + 1)
];
};
const orderByLuminance = (rgbValues) => {
const calculateLuminance = (p) => 0.2126 * p.r + 0.7152 * p.g + 0.0722 * p.b;
return rgbValues.sort(
(p1, p2) => calculateLuminance(p2) - calculateLuminance(p1)
);
};
let palette = [];
let dominantColor = void 0;
let secondaryColor = void 0;
let accentColor = void 0;
let accessibility = void 0;
try {
log(
`Starting palette extraction. Options: deltaEThreshold=${deltaEThreshold}, paletteSize=${paletteSize}`
);
let rgbArray = buildRgb(imageData, 5);
log(`Built RGB array of ${rgbArray.length} sample pixels.`);
let quantColors = quantization(rgbArray, 0);
log(`Quantized colors down to ${quantColors.length} primary colors.`);
let orderedByColor = orderByLuminance(quantColors);
log("Sorted quantized colors by luminance.");
const distinctColors = [];
if (orderedByColor.length > 0) {
distinctColors.push(orderedByColor[0]);
const labCache = [rgbToLab(orderedByColor[0])];
for (let i = 1; i < orderedByColor.length; i++) {
if (distinctColors.length >= paletteSize) {
log(`Reached target palette size of ${paletteSize}. Stopping.`);
break;
}
const currentLab = rgbToLab(orderedByColor[i]);
let isDistinct = true;
for (const existingLab of labCache) {
if (deltaE(currentLab, existingLab) < deltaEThreshold) {
isDistinct = false;
break;
}
}
if (isDistinct) {
distinctColors.push(orderedByColor[i]);
labCache.push(currentLab);
}
}
}
log(
`Filtered down to ${distinctColors.length} perceptually distinct colors.`
);
palette = distinctColors.map(rgbToHex);
log(`Final palette generated: [${palette.join(", ")}]`);
if (palette.length > 0) {
const paletteRgb = palette.map((hex) => ({ hex, rgb: hexToRgb(hex) }));
const paletteHsl = paletteRgb.map((c) => ({ ...c, hsl: rgbToHsl(c.rgb) }));
const getVibrancy = (hsl) => hsl.s * (1 - Math.abs(hsl.l - 0.5));
paletteHsl.sort((a, b) => getVibrancy(b.hsl) - getVibrancy(a.hsl));
const dominantInfo = paletteHsl[0];
dominantColor = dominantInfo.hex;
log(`Selected most vibrant color as dominantColor: ${dominantColor}`);
const remainingColorsHsl = paletteHsl.filter(
(c) => c.hex !== dominantColor
);
const dominantLuminance = dominantInfo.hsl.l;
const remainingByLuminance = [...remainingColorsHsl].sort(
(a, b) => b.hsl.l - a.hsl.l
);
const remainingBySaturation = [...remainingColorsHsl].sort(
(a, b) => a.hsl.s - b.hsl.s
);
if (dominantLuminance > 0.65) {
log(`Dominant color is BRIGHT.`);
secondaryColor = (_a = remainingByLuminance[0]) == null ? void 0 : _a.hex;
log(`-> Secondary (brightest remaining): ${secondaryColor}`);
} else if (dominantLuminance < 0.35) {
log(`Dominant color is DARK.`);
secondaryColor = (_b = remainingBySaturation[0]) == null ? void 0 : _b.hex;
log(`-> Secondary (most muted): ${secondaryColor}`);
} else {
log(`Dominant color is MUTED.`);
secondaryColor = (_c = remainingBySaturation[0]) == null ? void 0 : _c.hex;
log(`-> Secondary (most muted): ${secondaryColor}`);
}
const accentCandidates = paletteHsl.filter(
(c) => c.hex !== dominantColor && c.hex !== secondaryColor
);
if (accentCandidates.length > 0) {
const candidatesWithScores = accentCandidates.map((c) => ({
...c,
vibrancyScore: c.hsl.s * (1 - Math.abs(c.hsl.l - 0.5)),
// Higher for balanced saturation/luminance
contrastWithDominant: getContrastRatio(c.rgb, dominantInfo.rgb)
}));
candidatesWithScores.sort((a, b) => {
const aHasGoodContrast = a.contrastWithDominant >= 3;
const bHasGoodContrast = b.contrastWithDominant >= 3;
if (aHasGoodContrast && !bHasGoodContrast) return -1;
if (!aHasGoodContrast && bHasGoodContrast) return 1;
return b.vibrancyScore - a.vibrancyScore;
});
accentColor = (_d = candidatesWithScores[0]) == null ? void 0 : _d.hex;
log(
`-> Accent (most striking with good contrast): ${accentColor} (vibrancy: ${(_e = candidatesWithScores[0]) == null ? void 0 : _e.vibrancyScore.toFixed(3)}, contrast: ${(_f = candidatesWithScores[0]) == null ? void 0 : _f.contrastWithDominant.toFixed(2)})`
);
}
const dominantRgb = hexToRgb(dominantColor);
if (dominantRgb) {
const black = { r: 0, g: 0, b: 0 };
const white = { r: 255, g: 255, b: 255 };
const contrastWithBlack = getContrastRatio(dominantRgb, black);
const contrastWithWhite = getContrastRatio(dominantRgb, white);
if (contrastWithBlack > contrastWithWhite) {
accessibility = {
color: "#000000",
contrastRatio: contrastWithBlack
};
} else {
accessibility = {
color: "#FFFFFF",
contrastRatio: contrastWithWhite
};
}
log(
`Accessibility color is ${accessibility.color} with contrast ${accessibility.contrastRatio.toFixed(2)}.`
);
}
}
} catch (e) {
log(`ERROR during palette extraction: ${e.message || "Unknown error"}`);
}
return {
palette,
dominantColor,
secondaryColor,
accentColor,
accessibility,
logs
};
}
// src/functions/extractImages.ts
async function extractImages(page, options) {
console.log("Extracting images from page...");
const pageUrl = page.url();
const enableColorExtraction = options == null ? void 0 : options.enableColorExtraction;
const extractPaletteFnString = extractPaletteFromImageData.toString();
const images = await page.$$eval(
"img",
(imgs, pageUrl2, enableColorExtraction2, _extractPaletteString) => {
const _extractPalette = new Function(
`return ${_extractPaletteString}`
)();
return imgs.map((img) => {
var _a;
let palette = void 0;
let dominantColor = void 0;
let secondaryColor = void 0;
let accentColor = void 0;
let accessibility = void 0;
const logs = [];
if (enableColorExtraction2) {
logs.push(`Color extraction enabled for image: ${img.src}`);
try {
const canvas = document.createElement("canvas");
canvas.width = img.naturalWidth || img.width;
canvas.height = img.naturalHeight || img.height;
if (canvas.width && canvas.height) {
logs.push(
`Created canvas with dimensions: ${canvas.width}x${canvas.height}`
);
const ctx = canvas.getContext("2d");
if (ctx) {
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
const imageData = ctx.getImageData(
0,
0,
canvas.width,
canvas.height
).data;
const result2 = _extractPalette(imageData);
palette = result2.palette;
dominantColor = result2.dominantColor;
secondaryColor = result2.secondaryColor;
accentColor = result2.accentColor;
accessibility = result2.accessibility;
if (result2.logs) {
logs.push(...result2.logs);
}
} else {
logs.push("Failed to get 2D context from canvas.");
}
} else {
logs.push(
`Skipping color extraction: Invalid image dimensions (${img.naturalWidth}x${img.naturalHeight})`
);
}
} catch (e) {
logs.push(
`ERROR during canvas operations: ${e.message || "Unknown error"}`
);
}
}
const parentAnchor = img.closest("a");
const imgPermalink = parentAnchor ? parentAnchor.href : pageUrl2;
const result = {
src: img.getAttribute("src"),
srcset: img.getAttribute("srcset"),
responsiveImages: ((_a = img.getAttribute("srcset")) == null ? void 0 : _a.split(",").map((src) => {
const trimmedSrc = src.trim();
if (!trimmedSrc) {
return null;
}
const parts = trimmedSrc.split(/\s+/);
const url = parts[0];
const size = parts.length > 1 ? parts[1] : "";
if (!url) {
return null;
}
return { url, size };
}).filter(
(item) => item !== null
)) || null,
alt: img.getAttribute("alt"),
imgPermalink,
pagePermalink: pageUrl2,
width: img.width || null,
height: img.height || null
};
if (enableColorExtraction2) {
result.dominantColor = dominantColor || null;
result.secondaryColor = secondaryColor || null;
result.accentColor = accentColor || null;
result.accessibility = accessibility || null;
result.palette = palette || null;
result.colorExtractionLogs = logs;
}
return result;
});
},
pageUrl,
Boolean(enableColorExtraction),
extractPaletteFnString
);
console.log(`Extracted ${images.length} images.`);
return images;
}
async function extractBackgroundImages(page) {
console.log("Deep extracting background images from page...");
const pageUrl = page.url();
const images = await page.evaluate((pageUrl2) => {
var _a, _b;
const allImages = [];
const elements = document.querySelectorAll('a[href^="/jobs/"]');
console.log(
`Found ${elements.length} elements to scan for background images.`
);
for (const element of Array.from(elements)) {
let backgroundImage = (_a = element.style) == null ? void 0 : _a.backgroundImage;
if (!backgroundImage || backgroundImage === "none") {
backgroundImage = window.getComputedStyle(element).getPropertyValue("background-image");
}
if (backgroundImage && backgroundImage !== "none") {
const anchor = element.closest("a");
const imgPermalink = anchor ? anchor.href : pageUrl2;
const width = element.clientWidth;
const height = element.clientHeight;
console.log(
`Processing element ${element.tagName} with background-image: ${backgroundImage}`
);
const imageSetRegex = /(-webkit-)?image-set\(([^)]+)\)/g;
let imageSetMatch;
let processedImageSet = false;
while ((imageSetMatch = imageSetRegex.exec(backgroundImage)) !== null) {
processedImageSet = true;
const imageSetContent = imageSetMatch[2];
if (imageSetContent) {
const responsiveImages = [];
const parts = imageSetContent.split(",");
parts.forEach((part) => {
const trimmedPart = part.trim();
const urlMatch2 = trimmedPart.match(/url\((['"]?)(.*?)\1\)/);
const sizeMatch = trimmedPart.match(/\s+([\d.]+)x/);
if (urlMatch2 && urlMatch2[2]) {
const url = urlMatch2[2];
if (url && !url.startsWith("data:")) {
responsiveImages.push({
url,
size: sizeMatch ? sizeMatch[1] + "x" : "1x"
});
}
}
});
if (responsiveImages.length > 0) {
allImages.push({
src: ((_b = responsiveImages[0]) == null ? void 0 : _b.url) || null,
srcset: responsiveImages.map((img) => `${img.url} ${img.size}`).join(", "),
responsiveImages,
alt: `Background image from ${element.tagName.toLowerCase()}`,
imgPermalink,
pagePermalink: pageUrl2,
width,
height
});
}
}
}
if (processedImageSet) {
continue;
}
const urlRegex = /url\((['"]?)(.*?)\1\)/g;
let urlMatch;
while ((urlMatch = urlRegex.exec(backgroundImage)) !== null) {
const url = urlMatch[2].trim();
if (url && !url.startsWith("data:")) {
if (!allImages.some(
(img) => img.src === url || img.responsiveImages && img.responsiveImages.some((r) => r.url === url)
)) {
allImages.push({
src: url,
srcset: null,
responsiveImages: null,
alt: `Background image from ${element.tagName.toLowerCase()}`,
imgPermalink,
pagePermalink: pageUrl2,
width,
height
});
}
}
}
}
}
return allImages;
}, pageUrl);
console.log(`Deep extracted ${images.length} background images.`);
return images;
}
async function extractImagesFromURL(options) {
try {
console.log(`Starting image extraction from URL: ${options.url}`);
const { browser, page } = await openPage(options);
console.log("Page has been opened.");
if (options.deepExtract) {
console.log("Scrolling page to trigger lazy loading...");
await page.evaluate(async () => {
const scrollableElement = document.getElementById("pageScroll") || document.body;
await new Promise((resolve) => {
let lastHeight = -1;
let steadyCount = 0;
let totalScrolls = 0;
const maxSteadyCount = 3;
const maxScrolls = 30;
const timer = setInterval(() => {
totalScrolls++;
const newHeight = scrollableElement.scrollHeight;
if (newHeight === lastHeight) {
steadyCount++;
} else {
steadyCount = 0;
}
if (steadyCount >= maxSteadyCount || totalScrolls >= maxScrolls) {
clearInterval(timer);
resolve();
return;
}
scrollableElement.scrollTo(0, newHeight);
lastHeight = newHeight;
}, 500);
});
});
console.log(
"Finished scrolling. Waiting for 5 seconds for content to load..."
);
await new Promise((resolve) => setTimeout(resolve, 5e3));
}
let images = await extractImages(page, {
enableColorExtraction: options.enableColorExtraction
});
if (options.deepExtract) {
const backgroundImages = await extractBackgroundImages(page);
images = images.concat(backgroundImages);
}
images.forEach((img) => {
if (img.colorExtractionLogs && img.colorExtractionLogs.length > 0) {
console.log(`--- Logs for image: ${img.src} ---`);
img.colorExtractionLogs.forEach((log) => console.log(log));
console.log("--- End of logs ---");
}
});
console.log("Closing browser session.");
await browser.close();
console.log("Browser has been closed.");
return images;
} catch (error) {
console.error("Error extracting images:", error);
throw error;
}
}
// src/functions/extractVideos.ts
async function extractVideos(page) {
console.log("Extracting videos from page...");
const pageUrl = page.url();
const videos = await page.$$eval(
"video",
(videos2, pageUrl2) => videos2.map((video) => {
const parentAnchor = video.closest("a");
const videoPermalink = parentAnchor ? parentAnchor.href : pageUrl2;
const sourceElements = Array.from(video.querySelectorAll("source"));
const sources = sourceElements.length > 0 ? sourceElements.map((source) => ({
src: source.src,
type: source.getAttribute("type")
})) : null;
return {
src: video.getAttribute("src"),
sources,
poster: video.getAttribute("poster"),
videoPermalink,
pagePermalink: pageUrl2,
width: video.videoWidth || video.width || null,
height: video.videoHeight || video.height || null
};
}),
pageUrl
);
console.log(`Extracted ${videos.length} videos.`);
return videos;
}
async function extractVideosFromURL(options) {
try {
console.log(`Starting video extraction from URL: ${options.url}`);
const { browser, page } = await openPage(options);
console.log("Page has been opened.");
if (options.deepExtract) {
console.log("Scrolling page to trigger lazy loading...");
await page.evaluate(async () => {
const scrollableElement = document.getElementById("pageScroll") || document.body;
await new Promise((resolve) => {
let lastHeight = -1;
let steadyCount = 0;
let totalScrolls = 0;
const maxSteadyCount = 3;
const maxScrolls = 30;
const timer = setInterval(() => {
totalScrolls++;
const newHeight = scrollableElement.scrollHeight;
if (newHeight === lastHeight) {
steadyCount++;
} else {
steadyCount = 0;
}
if (steadyCount >= maxSteadyCount || totalScrolls >= maxScrolls) {
clearInterval(timer);
resolve();
return;
}
scrollableElement.scrollTo(0, newHeight);
lastHeight = newHeight;
}, 500);
});
});
console.log(
"Finished scrolling. Waiting for 5 seconds for content to load..."
);
await new Promise((resolve) => setTimeout(resolve, 5e3));
}
const videos = await extractVideos(page);
console.log("Closing browser session.");
await browser.close();
console.log("Browser has been closed.");
return videos;
} catch (error) {
console.error("Error extracting videos:", error);
throw error;
}
}
// src/puppeteerSLS.ts
var PuppeteerSLS = {
extractImagesFromURL,
extractLinksFromUrl,
extractWebpage,
openPage,
takeSnapShot
};
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
PuppeteerSLS,
extractBackgroundImages,
extractHTML,
extractImages,
extractImagesFromURL,
extractLinks,
extractLinksFromUrl,
extractToMarkdown,
extractVideos,
extractVideosFromURL,
extractWebpage,
openPage,
puppeteerLaunchProps,
takeSnapShot
});
//# sourceMappingURL=index.js.map