krabber
Version:
Easy to use webscraper
61 lines • 4.63 kB
JavaScript
import axios from 'axios';
import cleanDeep from 'clean-deep';
import { JSDOM } from 'jsdom';
export function Scrape(config) {
let ax;
const pageResults = [];
let finalResult;
if (!config.url) {
throw new Error('url is required');
}
if (config.axiosInstance) {
ax = config.axiosInstance;
}
else {
ax = axios.create({
transformResponse: (data) => new JSDOM(data),
});
}
const { select, paginate, options } = config;
const originalUrl = config.url;
async function _scrape(url, prev) {
const response = await ax(url, options);
const dom = response.data.window.document;
let res = {};
for (const key in select) {
// tslint:disable-next-line: prefer-conditional-expression
if (typeof select[key] === 'function') {
res[key] = await select[key]({ dom, res, prev, url });
}
else {
res[key] = select[key];
}
}
res = cleanDeep(res);
if (paginate) {
pageResults.push(res);
const args = { dom, res, prev, url };
const next = await paginate.next(args);
if (next) {
return _scrape(next, res);
}
const { map, reduce } = paginate;
if (map) {
finalResult = pageResults.map(map(args));
}
if (reduce && map) {
finalResult = finalResult.reduce(reduce(args), null);
}
if (reduce && !map) {
finalResult = pageResults.reduce(reduce(args), null);
}
if (finalResult) {
return finalResult;
}
return pageResults;
}
return res;
}
return _scrape(originalUrl, {});
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyYXBlLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi9zY3JhcGUudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUE0QyxNQUFNLE9BQU8sQ0FBQztBQUNqRSxPQUFPLFNBQVMsTUFBTSxZQUFZLENBQUM7QUFDbkMsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLE9BQU8sQ0FBQztBQWtDOUIsTUFBTSxVQUFVLE1BQU0sQ0FBQyxNQUFvQjtJQUN6QyxJQUFJLEVBQWlCLENBQUM7SUFDdEIsTUFBTSxXQUFXLEdBQVUsRUFBRSxDQUFDO0lBQzlCLElBQUksV0FBZ0IsQ0FBQztJQUVyQixJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsRUFBRTtRQUNmLE1BQU0sSUFBSSxLQUFLLENBQUMsaUJBQWlCLENBQUMsQ0FBQztLQUNwQztJQUVELElBQUksTUFBTSxDQUFDLGFBQWEsRUFBRTtRQUN4QixFQUFFLEdBQUcsTUFBTSxDQUFDLGFBQWEsQ0FBQztLQUMzQjtTQUFNO1FBQ0wsRUFBRSxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUM7WUFDaEIsaUJBQWlCLEVBQUUsQ0FBQyxJQUFJLEVBQUUsRUFBRSxDQUFDLElBQUksS0FBSyxDQUFDLElBQUksQ0FBQztTQUM3QyxDQUFDLENBQUM7S0FDSjtJQUVELE1BQU0sRUFBRSxNQUFNLEVBQUUsUUFBUSxFQUFFLE9BQU8sRUFBRSxHQUFHLE1BQU0sQ0FBQztJQUM3QyxNQUFNLFdBQVcsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDO0lBRS9CLEtBQUssVUFBVSxPQUFPLENBQUMsR0FBVyxFQUFFLElBQVM7UUFDM0MsTUFBTSxRQUFRLEdBQUcsTUFBTSxFQUFFLENBQUMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQ3hDLE1BQU0sR0FBRyxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLFFBQVEsQ0FBQztRQUMxQyxJQUFJLEdBQUcsR0FBUSxFQUFFLENBQUM7UUFFbEIsS0FBSyxNQUFNLEdBQUcsSUFBSSxNQUFNLEVBQUU7WUFDeEIsMERBQTBEO1lBQzFELElBQUksT0FBTyxNQUFNLENBQUMsR0FBRyxDQUFDLEtBQUssVUFBVSxFQUFFO2dCQUNyQyxHQUFHLENBQUMsR0FBRyxDQUFDLEdBQUcsTUFBTSxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUMsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLElBQUksRUFBRSxHQUFHLEVBQUUsQ0FBQyxDQUFDO2FBQ3ZEO2lCQUFNO2dCQUNMLEdBQUcsQ0FBQyxHQUFHLENBQUMsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUM7YUFDeEI7U0FDRjtRQUVELEdBQUcsR0FBRyxTQUFTLENBQUMsR0FBRyxDQUFDLENBQUM7UUFFckIsSUFBSSxRQUFRLEVBQUU7WUFDWixXQUFXLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBRXRCLE1BQU0sSUFBSSxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBRSxJQUFJLEVBQUUsR0FBRyxFQUFFLENBQUM7WUFFckMsTUFBTSxJQUFJLEdBQUcsTUFBTSxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQ3ZDLElBQUksSUFBSSxFQUFFO2dCQUNSLE9BQU8sT0FBTyxDQUFDLElBQUksRUFBRSxHQUFHLENBQUMsQ0FBQzthQUMzQjtZQUVELE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxFQUFFLEdBQUcsUUFBUSxDQUFDO1lBQ2pDLElBQUksR0FBRyxFQUFFO2dCQUNQLFdBQVcsR0FBRyxXQUFXLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO2FBQzFDO1lBQ0QsSUFBSSxNQUFNLElBQUksR0FBRyxFQUFFO2dCQUNqQixXQUFXLEdBQUcsV0FBVyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEVBQUUsSUFBSSxDQUFDLENBQUM7YUFDdEQ7WUFDRCxJQUFJLE1BQU0sSUFBSSxDQUFDLEdBQUcsRUFBRTtnQkFDbEIsV0FBVyxHQUFHLFdBQVcsQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxFQUFFLElBQUksQ0FBQyxDQUFDO2FBQ3REO1lBRUQsSUFBSSxXQUFXLEVBQUU7Z0JBQ2YsT0FBTyxXQUFXLENBQUM7YUFDcEI7WUFFRCxPQUFPLFdBQVcsQ0FBQztTQUNwQjtRQUVELE9BQU8sR0FBRyxDQUFDO0lBQ2IsQ0FBQztJQUVELE9BQU8sT0FBTyxDQUFDLFdBQVcsRUFBRSxFQUFFLENBQUMsQ0FBQztBQUNsQyxDQUFDIn0=