krabber
Version:
Easy to use webscraper
68 lines • 4.91 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Scrape = void 0;
const axios_1 = __importDefault(require("axios"));
const clean_deep_1 = __importDefault(require("clean-deep"));
const jsdom_1 = require("jsdom");
function Scrape(config) {
let ax;
const pageResults = [];
let finalResult;
if (!config.url) {
throw new Error('url is required');
}
if (config.axiosInstance) {
ax = config.axiosInstance;
}
else {
ax = axios_1.default.create({
transformResponse: (data) => new jsdom_1.JSDOM(data),
});
}
const { select, paginate, options } = config;
const originalUrl = config.url;
async function _scrape(url, prev) {
const response = await ax(url, options);
const dom = response.data.window.document;
let res = {};
for (const key in select) {
// tslint:disable-next-line: prefer-conditional-expression
if (typeof select[key] === 'function') {
res[key] = await select[key]({ dom, res, prev, url });
}
else {
res[key] = select[key];
}
}
res = clean_deep_1.default(res);
if (paginate) {
pageResults.push(res);
const args = { dom, res, prev, url };
const next = await paginate.next(args);
if (next) {
return _scrape(next, res);
}
const { map, reduce } = paginate;
if (map) {
finalResult = pageResults.map(map(args));
}
if (reduce && map) {
finalResult = finalResult.reduce(reduce(args), null);
}
if (reduce && !map) {
finalResult = pageResults.reduce(reduce(args), null);
}
if (finalResult) {
return finalResult;
}
return pageResults;
}
return res;
}
return _scrape(originalUrl, {});
}
exports.Scrape = Scrape;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NyYXBlLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi9zY3JhcGUudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7O0FBQUEsa0RBQWlFO0FBQ2pFLDREQUFtQztBQUNuQyxpQ0FBOEI7QUFrQzlCLFNBQWdCLE1BQU0sQ0FBQyxNQUFvQjtJQUN6QyxJQUFJLEVBQWlCLENBQUM7SUFDdEIsTUFBTSxXQUFXLEdBQVUsRUFBRSxDQUFDO0lBQzlCLElBQUksV0FBZ0IsQ0FBQztJQUVyQixJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsRUFBRTtRQUNmLE1BQU0sSUFBSSxLQUFLLENBQUMsaUJBQWlCLENBQUMsQ0FBQztLQUNwQztJQUVELElBQUksTUFBTSxDQUFDLGFBQWEsRUFBRTtRQUN4QixFQUFFLEdBQUcsTUFBTSxDQUFDLGFBQWEsQ0FBQztLQUMzQjtTQUFNO1FBQ0wsRUFBRSxHQUFHLGVBQUssQ0FBQyxNQUFNLENBQUM7WUFDaEIsaUJBQWlCLEVBQUUsQ0FBQyxJQUFJLEVBQUUsRUFBRSxDQUFDLElBQUksYUFBSyxDQUFDLElBQUksQ0FBQztTQUM3QyxDQUFDLENBQUM7S0FDSjtJQUVELE1BQU0sRUFBRSxNQUFNLEVBQUUsUUFBUSxFQUFFLE9BQU8sRUFBRSxHQUFHLE1BQU0sQ0FBQztJQUM3QyxNQUFNLFdBQVcsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDO0lBRS9CLEtBQUssVUFBVSxPQUFPLENBQUMsR0FBVyxFQUFFLElBQVM7UUFDM0MsTUFBTSxRQUFRLEdBQUcsTUFBTSxFQUFFLENBQUMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQ3hDLE1BQU0sR0FBRyxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLFFBQVEsQ0FBQztRQUMxQyxJQUFJLEdBQUcsR0FBUSxFQUFFLENBQUM7UUFFbEIsS0FBSyxNQUFNLEdBQUcsSUFBSSxNQUFNLEVBQUU7WUFDeEIsMERBQTBEO1lBQzFELElBQUksT0FBTyxNQUFNLENBQUMsR0FBRyxDQUFDLEtBQUssVUFBVSxFQUFFO2dCQUNyQyxHQUFHLENBQUMsR0FBRyxDQUFDLEdBQUcsTUFBTSxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUMsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLElBQUksRUFBRSxHQUFHLEVBQUUsQ0FBQyxDQUFDO2FBQ3ZEO2lCQUFNO2dCQUNMLEdBQUcsQ0FBQyxHQUFHLENBQUMsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUM7YUFDeEI7U0FDRjtRQUVELEdBQUcsR0FBRyxvQkFBUyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1FBRXJCLElBQUksUUFBUSxFQUFFO1lBQ1osV0FBVyxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztZQUV0QixNQUFNLElBQUksR0FBRyxFQUFFLEdBQUcsRUFBRSxHQUFHLEVBQUUsSUFBSSxFQUFFLEdBQUcsRUFBRSxDQUFDO1lBRXJDLE1BQU0sSUFBSSxHQUFHLE1BQU0sUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztZQUN2QyxJQUFJLElBQUksRUFBRTtnQkFDUixPQUFPLE9BQU8sQ0FBQyxJQUFJLEVBQUUsR0FBRyxDQUFDLENBQUM7YUFDM0I7WUFFRCxNQUFNLEVBQUUsR0FBRyxFQUFFLE1BQU0sRUFBRSxHQUFHLFFBQVEsQ0FBQztZQUNqQyxJQUFJLEdBQUcsRUFBRTtnQkFDUCxXQUFXLEdBQUcsV0FBVyxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQzthQUMxQztZQUNELElBQUksTUFBTSxJQUFJLEdBQUcsRUFBRTtnQkFDakIsV0FBVyxHQUFHLFdBQVcsQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxFQUFFLElBQUksQ0FBQyxDQUFDO2FBQ3REO1lBQ0QsSUFBSSxNQUFNLElBQUksQ0FBQyxHQUFHLEVBQUU7Z0JBQ2xCLFdBQVcsR0FBRyxXQUFXLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsRUFBRSxJQUFJLENBQUMsQ0FBQzthQUN0RDtZQUVELElBQUksV0FBVyxFQUFFO2dCQUNmLE9BQU8sV0FBVyxDQUFDO2FBQ3BCO1lBRUQsT0FBTyxXQUFXLENBQUM7U0FDcEI7UUFFRCxPQUFPLEdBQUcsQ0FBQztJQUNiLENBQUM7SUFFRCxPQUFPLE9BQU8sQ0FBQyxXQUFXLEVBQUUsRUFBRSxDQUFDLENBQUM7QUFDbEMsQ0FBQztBQXBFRCx3QkFvRUMifQ==