chromancer
Version:
A powerful command-line interface for automating Chrome browser using Playwright. Perfect for web scraping, automation, testing, and browser workflows.
370 lines (369 loc) • 16.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
const core_1 = require("@oclif/core");
const base_js_1 = require("../base.js");
const fs_1 = require("fs");
const path = tslib_1.__importStar(require("path"));
class Export extends base_js_1.BaseCommand {
static description = 'Export page content, data, or resources';
static examples = [
'<%= config.bin %> <%= command.id %> --format html --output page.html',
'<%= config.bin %> <%= command.id %> --format json --selector "table#data"',
'<%= config.bin %> <%= command.id %> --format csv --selector "table"',
'<%= config.bin %> <%= command.id %> --format markdown --output content.md',
'<%= config.bin %> <%= command.id %> --all-resources --output-dir ./export',
];
static flags = {
...base_js_1.BaseCommand.baseFlags,
format: core_1.Flags.string({
char: 'f',
description: 'Export format',
options: ['html', 'json', 'csv', 'markdown', 'text'],
default: 'html',
}),
selector: core_1.Flags.string({
char: 's',
description: 'CSS selector to export specific element',
}),
output: core_1.Flags.string({
char: 'o',
description: 'Output file path',
}),
'output-dir': core_1.Flags.string({
description: 'Output directory for resources',
default: './export',
}),
'all-resources': core_1.Flags.boolean({
description: 'Export all page resources (images, scripts, styles)',
default: false,
}),
'include-styles': core_1.Flags.boolean({
description: 'Include computed styles in HTML export',
default: false,
}),
};
async run() {
const { flags } = await this.parse(Export);
await this.connectToChrome(flags.port, flags.host, flags.launch, flags.profile, flags.headless, flags.verbose, flags.keepOpen);
if (!this.page) {
this.error('Failed to connect to Chrome');
}
await this.exportContent(this.page);
}
async exportContent(page) {
const { flags } = await this.parse(Export);
let content;
switch (flags.format) {
case 'html':
content = await this.exportHTML(page, flags.selector, flags['include-styles']);
break;
case 'json':
content = await this.exportJSON(page, flags.selector);
break;
case 'csv':
content = await this.exportCSV(page, flags.selector);
break;
case 'markdown':
content = await this.exportMarkdown(page, flags.selector);
break;
case 'text':
content = await this.exportText(page, flags.selector);
break;
default:
this.error(`Unsupported format: ${flags.format}`);
}
// Export resources if requested
if (flags['all-resources']) {
await this.exportResources(page, flags['output-dir']);
}
// Save content
const outputPath = flags.output || `export.${flags.format}`;
await fs_1.promises.writeFile(outputPath, content);
this.log(`✅ Exported to: ${outputPath}`);
}
async exportHTML(page, selector, includeStyles) {
if (selector) {
return await page.evaluate(({ sel, styles }) => {
const element = document.querySelector(sel);
if (!element)
throw new Error(`Element not found: ${sel}`);
if (styles) {
// Clone element and inline styles
const clone = element.cloneNode(true);
const allElements = clone.querySelectorAll('*');
allElements.forEach((el) => {
const computed = window.getComputedStyle(el);
el.setAttribute('style', computed.cssText);
});
return clone.outerHTML;
}
return element.outerHTML;
}, { sel: selector, styles: includeStyles });
}
return await page.content();
}
async exportJSON(page, selector) {
const data = await page.evaluate((sel) => {
if (sel) {
const element = document.querySelector(sel);
if (!element)
throw new Error(`Element not found: ${sel}`);
// Special handling for tables
if (element.tagName === 'TABLE') {
const headers = [];
const rows = [];
// Get headers
element.querySelectorAll('thead th').forEach((th) => {
headers.push(th.textContent?.trim() || '');
});
// Get rows
element.querySelectorAll('tbody tr').forEach((tr) => {
const row = {};
tr.querySelectorAll('td').forEach((td, i) => {
const key = headers[i] || `col${i}`;
row[key] = td.textContent?.trim() || '';
});
rows.push(row);
});
return rows;
}
// Try to extract data attributes
const dataAttrs = {};
Array.from(element.attributes).forEach((attr) => {
if (attr.name.startsWith('data-')) {
const key = attr.name.replace('data-', '');
try {
dataAttrs[key] = JSON.parse(attr.value);
}
catch {
dataAttrs[key] = attr.value;
}
}
});
return {
text: element.textContent?.trim(),
attributes: dataAttrs,
html: element.innerHTML,
};
}
// Export all data elements
const allData = [];
document.querySelectorAll('[data-export], [data-json], table').forEach((el) => {
if (el.tagName === 'TABLE') {
// Handle tables as above
const headers = [];
const rows = [];
el.querySelectorAll('thead th').forEach((th) => {
headers.push(th.textContent?.trim() || '');
});
el.querySelectorAll('tbody tr').forEach((tr) => {
const row = {};
tr.querySelectorAll('td').forEach((td, i) => {
const key = headers[i] || `col${i}`;
row[key] = td.textContent?.trim() || '';
});
rows.push(row);
});
allData.push({ type: 'table', data: rows });
}
else {
// Handle data elements
const data = el.getAttribute('data-json') || el.getAttribute('data-export');
if (data) {
try {
allData.push(JSON.parse(data));
}
catch {
allData.push(data);
}
}
}
});
return allData;
}, selector);
return JSON.stringify(data, null, 2);
}
async exportCSV(page, selector) {
const tableSelector = selector || 'table';
return await page.evaluate((sel) => {
const table = document.querySelector(sel);
if (!table || table.tagName !== 'TABLE') {
throw new Error('No table found with selector: ' + sel);
}
const rows = [];
// Get headers
const headers = [];
table.querySelectorAll('thead th').forEach((th) => {
headers.push(`"${th.textContent?.trim().replace(/"/g, '""') || ''}"`);
});
if (headers.length > 0) {
rows.push(headers.join(','));
}
// Get data rows
table.querySelectorAll('tbody tr').forEach((tr) => {
const cells = [];
tr.querySelectorAll('td').forEach((td) => {
cells.push(`"${td.textContent?.trim().replace(/"/g, '""') || ''}"`);
});
rows.push(cells.join(','));
});
return rows.join('\n');
}, tableSelector);
}
async exportMarkdown(page, selector) {
return await page.evaluate((sel) => {
const element = sel ? document.querySelector(sel) : document.body;
if (!element)
throw new Error(`Element not found: ${sel}`);
// Simple HTML to Markdown conversion
function htmlToMarkdown(el) {
let md = '';
for (const node of Array.from(el.childNodes)) {
if (node.nodeType === Node.TEXT_NODE) {
md += node.textContent;
}
else if (node.nodeType === Node.ELEMENT_NODE) {
const elem = node;
const tag = elem.tagName.toLowerCase();
switch (tag) {
case 'h1':
md += `# ${elem.textContent?.trim()}\n\n`;
break;
case 'h2':
md += `## ${elem.textContent?.trim()}\n\n`;
break;
case 'h3':
md += `### ${elem.textContent?.trim()}\n\n`;
break;
case 'h4':
md += `#### ${elem.textContent?.trim()}\n\n`;
break;
case 'h5':
md += `##### ${elem.textContent?.trim()}\n\n`;
break;
case 'h6':
md += `###### ${elem.textContent?.trim()}\n\n`;
break;
case 'p':
md += `${elem.textContent?.trim()}\n\n`;
break;
case 'a':
md += `[${elem.textContent?.trim()}](${elem.getAttribute('href')})`;
break;
case 'img':
md += `})`;
break;
case 'strong':
case 'b':
md += `**${elem.textContent?.trim()}**`;
break;
case 'em':
case 'i':
md += `*${elem.textContent?.trim()}*`;
break;
case 'code':
md += `\`${elem.textContent?.trim()}\``;
break;
case 'pre':
md += `\`\`\`\n${elem.textContent?.trim()}\n\`\`\`\n\n`;
break;
case 'ul':
case 'ol':
const items = Array.from(elem.querySelectorAll('li'));
items.forEach((li, i) => {
const prefix = tag === 'ol' ? `${i + 1}. ` : '- ';
md += `${prefix}${li.textContent?.trim()}\n`;
});
md += '\n';
break;
case 'blockquote':
md += `> ${elem.textContent?.trim()}\n\n`;
break;
case 'hr':
md += '---\n\n';
break;
case 'br':
md += '\n';
break;
case 'table':
// Convert table to markdown
const headers = Array.from(elem.querySelectorAll('thead th'));
if (headers.length > 0) {
md += '| ' + headers.map(th => th.textContent?.trim()).join(' | ') + ' |\n';
md += '| ' + headers.map(() => '---').join(' | ') + ' |\n';
}
elem.querySelectorAll('tbody tr').forEach((tr) => {
const cells = Array.from(tr.querySelectorAll('td'));
md += '| ' + cells.map(td => td.textContent?.trim()).join(' | ') + ' |\n';
});
md += '\n';
break;
default:
md += htmlToMarkdown(elem);
}
}
}
return md;
}
return htmlToMarkdown(element);
}, selector);
}
async exportText(page, selector) {
if (selector) {
return await page.evaluate((sel) => {
const element = document.querySelector(sel);
if (!element)
throw new Error(`Element not found: ${sel}`);
return element.textContent?.trim() || '';
}, selector);
}
return await page.evaluate(() => document.body.textContent?.trim() || '');
}
async exportResources(page, outputDir) {
const { flags } = await this.parse(Export);
await fs_1.promises.mkdir(outputDir, { recursive: true });
// Get all resource URLs
const resources = await page.evaluate(() => {
const urls = new Set();
// Images
document.querySelectorAll('img').forEach((img) => {
if (img.src)
urls.add(img.src);
});
// Stylesheets
document.querySelectorAll('link[rel="stylesheet"]').forEach((link) => {
if (link.href)
urls.add(link.href);
});
// Scripts
document.querySelectorAll('script[src]').forEach((script) => {
if (script.src)
urls.add(script.src);
});
return Array.from(urls);
});
this.log(`📦 Exporting ${resources.length} resources...`);
// Download each resource
for (const url of resources) {
try {
const response = await page.request.get(url);
if (response.ok()) {
const urlObj = new URL(url);
const filename = path.basename(urlObj.pathname) || 'index.html';
const filepath = path.join(outputDir, filename);
const buffer = await response.body();
await fs_1.promises.writeFile(filepath, buffer);
if (flags.verbose) {
this.logVerbose(`Downloaded: ${filename}`);
}
}
}
catch (error) {
this.warn(`Failed to download: ${url}`);
}
}
this.log(`✅ Resources exported to: ${outputDir}`);
}
}
exports.default = Export;