puppeteer-search-scraper
Version:
Scrape Search Engines using Puppeteer
444 lines (380 loc) • 16.2 kB
JavaScript
;
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
var puppeteer = require('puppeteer'),
devices = puppeteer.devices,
fs = require("fs");
var SearchScraper = function () {
_createClass(SearchScraper, [{
key: "log",
value: function log() {
var _console;
(_console = console).log.apply(_console, arguments);
}
/**
* register
*
* @name register
* @function
* @static
* @param {Object} c The SearchScraper options along with the `name` of the scraper.
* @returns {SearchScraper} The instance of the scraper.
*/
}], [{
key: "register",
value: function register(c) {
if (this._scrapers[c.name]) {
console.warn("There was already registered a scraper with this name: " + c.name);
}
return this._scrapers[c.name] = new SearchScraper(c);
}
/**
* getScraper
*
* @name getScraper
* @function
* @static
* @param {String} name The name of the scraper
* @returns {SearchScraper} The instance of the scraper.
*/
}, {
key: "getScraper",
value: function getScraper(name) {
return this._scrapers[name];
}
/**
* configure
*
* @name configure
* @function
* @static
* @param {Array} conf An array containing:
*/
}, {
key: "configure",
value: function configure(conf) {
if (!Array.isArray(conf)) {
conf = [conf];
}
return conf.map(function (c) {
return SearchScraper.register(c);
});
}
}, {
key: "search",
value: function search(term, options) {
var scr = SearchScraper.getScraper(options.engine);
return scr.search(term, options);
}
}, {
key: "trySearch",
value: function trySearch(term, options, limit) {
var scr = SearchScraper.getScraper(options.engine);
return scr.search(term, options, limit);
}
/**
* puppeteerGoogleScraper
* Scrape Google using Puppeteer
*
* @name puppeteerGoogleScraper
* @function
* @param {String} term The term to search.
* @param {Object} options An object containing:
*
* - `limit` (Number): The limit of the results (default: 100)
* - `headless` (Boolean): Whether the browser should be headless or not.
*
* @returns {Promise} A promise resolving with an array of elements containing:
*
* - `title` (String)
* - `url` (String)
*/
}]);
function SearchScraper(options) {
_classCallCheck(this, SearchScraper);
this.options = Object.assign({
limit: 100,
page_limit: 10,
headless: true,
debugDir: "",
searchUrl: "https://google.com",
selectors: {},
onFileWrite: function onFileWrite(path, content) {}
}, options);
}
_createClass(SearchScraper, [{
key: "trySearch",
value: async function trySearch(term, _options) {
var times = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 10;
for (var i = 0; i < times; ++i) {
try {
var res = await this.search(term, _options);
if (res.length) {
return res;
} else {
this.log("No data for " + term + " search term.");
}
} catch (e) {
this.log("Failed to search " + term + " but trying again.");
this.log(e.stack);
}
}
return [];
}
}, {
key: "search",
value: async function search(term, _options) {
var _this = this;
var options = Object.assign({}, this.options, _options);
options.selectors.result = options.selectors.result || {};
this.log("Loading the browser", options);
var browser = await puppeteer.launch({
headless: options.headless,
args: ['--no-sandbox', '--disable-setuid-sandbox']
});
// Debug function
var saveHtmlAndImg = async function saveHtmlAndImg(index) {
if (!options.debugDir) {
return;
}
try {
_this.log("Saving " + index);
var p = options.debugDir + "/" + index;
_this.log(">> ", p);
await page.screenshot({ path: p + ".png" });
_this.log("Screenshot saved.");
var html = await page.$eval("html", function (e) {
return e.outerHTML;
});
_this.log("HTML Saved");
var filePath = p + ".html";
fs.writeFileSync(filePath, html);
_this.log(">> File written.");
options.onFileWrite(filePath, html);
_this.log(">> after file write");
} catch (e) {
console.error("Failed to save HTML and image, but continuing: " + index);
console.error(e.stack);
}
};
this.log("Launching the page");
var page = await browser.newPage();
page.setRequestInterception(true);
var inflight = 0;
page.on('request', function (request) {
setTimeout(function () {
if (!request.is_finished) {
page.emit('requestfailed', request);
}
}, 7000);
inflight += 1;
request.continue();
});
page.on('requestfinished', function (request) {
request.is_finished = true;
inflight -= 1;
});
page.on('requestfailed', function (request) {
request.is_finished = true;
inflight -= 1;
});
var sleep = function sleep(timeout) {
return new Promise(function (resolve) {
return setTimeout(resolve, timeout);
});
};
var wait = async function wait(_ref) {
var waitUntil = _ref.waitUntil;
var maxIdle = waitUntil === 'networkidle0' ? 0 : 2;
while (inflight > maxIdle) {
await sleep(100);
}
await sleep(500);
if (inflight > maxIdle) {
await wait({ waitUntil: waitUntil });
}
};
if (options.device) {
var simulator = devices[options.device] || options.device;
await page.emulate(simulator);
}
await page.goto(options.searchUrl);
await saveHtmlAndImg(1);
var acceptTerms = async function acceptTerms() {
await sleep(1000);
if (options.selectors.accept_terms_button) {
if (typeof options.selectors.accept_terms_button === "function") {
try {
await sleep(2000);
await page.evaluate(options.selectors.accept_terms_button);
} catch (e) {
console.error(e);
}
} else {
try {
await page.click(options.selectors.accept_terms_button, { delay: 20 });
await sleep(1000);
await page.click(options.selectors.accept_terms_button, { delay: 20 });
} catch (e) {
console.error(e);
}
}
}
await sleep(1000);
};
this.log("Waiting for the input");
if (options.selectors.search_box) {
var searchInputSelector = options.selectors.search_box;
await page.waitForTimeout(2000);
await saveHtmlAndImg(2);
this.log("Accepting terms");
await acceptTerms();
await saveHtmlAndImg(3);
await page.waitForSelector(searchInputSelector);
this.log("Typing the search term");
await page.type(searchInputSelector, term, { delay: 100 }); // Types slower, like a user
await saveHtmlAndImg(4);
await page.keyboard.press('Enter');
await wait("networkidle0");
this.log("Waiting for the response");
await saveHtmlAndImg(5);
}
this.log(">>> After Searching.");
var currentPage = 0;
var nextIndex = 5;
var allLinks = [];
var doSync = async function doSync() {
_this.log(">> Do Sync", currentPage);
if (++currentPage > options.page_limit) {
return;
}
_this.log("Scraping results' page: " + currentPage);
_this.log("Random timeout");
await page.waitForTimeout(Math.floor(1000 + Math.random() * 2000));
await saveHtmlAndImg(++nextIndex);
_this.log("Waiting for the page results");
var resultSelector = options.selectors.result_items;
try {
await page.waitForSelector(resultSelector);
} catch (e) {
_this.log("Did not get the results after waiting. Perhaps there are no results returned by the search engine;.");
return;
}
_this.log("Parsing the page results");
var newLinks = await page.evaluate(options.scrape_fn || function (opts) {
var resultSelector = opts.selectors.result_items,
titleSelector = opts.selectors.result.title,
linkSelector = opts.selectors.result.link,
descriptionSelector = opts.selectors.result.description;
return [].slice.apply(document.querySelectorAll(resultSelector)).map(function (c) {
var newItem = {
title: ((titleSelector ? c.querySelector(titleSelector) : c) || { textContent: null }).textContent,
url: ((linkSelector ? c.querySelector(linkSelector) : c) || { href: null }).href,
description: ((descriptionSelector ? c.querySelector(descriptionSelector) : c) || { textContent: null }).textContent
};
c.remove();
return newItem;
});
}, options);
options.selectors.result.filter = options.selectors.result.filter || function (c) {
return c.title !== null && /^https?:\/\//.test(c.url);
};
if (options.selectors.result.filter) {
newLinks = newLinks.filter(options.selectors.result.filter);
}
// Hook for results
if (options.post_results_found) {
await options.post_results_found(page, newLinks);
}
allLinks.push.apply(allLinks, newLinks);
if (options.filter) {
allLinks = options.filter(allLinks);
}
allLinks = Object.values(allLinks.reduce(function (acc, c) {
acc[c.url] = c;
return acc;
}, {}));
if (allLinks.length > options.limit) {
return allLinks;
}
if (options.selectors.next_page) {
_this.log("Random timeout");
await page.waitForTimeout(Math.floor(1000 + Math.random() * 2000));
var existsNext = await page.$(options.selectors.next_page);
if (!existsNext) {
return allLinks;
}
_this.log("Random timeout");
await page.waitForTimeout(Math.floor(1000 + Math.random() * 2000));
_this.log("Going to the next page");
await page.click(options.selectors.next_page, { delay: 20 });
} else if (options.selectors.load_more) {
var hasLoadmore = await page.$(options.selectors.load_more);
if (!hasLoadmore) {
return;
}
_this.log("Random timeout");
await page.waitForTimeout(Math.floor(1000 + Math.random() * 2000));
_this.log("Loading more, until the number of needed results is hit.");
await page.click(options.selectors.load_more, { delay: 20 });
}
await wait("networkidle0");
await doSync();
};
await doSync();
this.log("Closing the browser");
try {
await browser.close();
} catch (e) {}
return allLinks.slice(0, options.limit);
}
}]);
return SearchScraper;
}();
// Scrapers and Selectors
var clickAcceptAllBtn = function clickAcceptAllBtn() {
var btns = [].slice.call(document.querySelectorAll("button")).filter(function (c) {
return (/accept/i.test(c.textContent)
);
});
btns.forEach(function (c) {
return c.click();
});
};
SearchScraper._scrapers = {};
SearchScraper.Selectors = {
GOOGLE: {
search_box: "form[action='/search'] textarea, form[action='/search'] input",
result_items: "#search [data-hveid], #search [data-ved]",
next_page: "#pnnext",
result: {
title: "h3",
link: "a[onmousedown], a[ping], a[data-jsarwt]",
description: "div[data-sncf='1']"
},
accept_terms_button: clickAcceptAllBtn
},
GOOGLE_MOBILE: {
search_box: "form[action='/search'] textarea, form[action='/search'] input",
result_items: "[data-hveid]",
load_more: "a[jsname][data-ved][jsaction][aria-label]",
result: {
title: "[aria-level='3']",
link: "a[ping]",
description: "div[data-sncf='1']"
},
accept_terms_button: clickAcceptAllBtn
},
BING: {
search_box: "form[action='/search'] input[type='text']",
result_items: "#b_results .b_algo",
next_page: "#b_results > li.b_pag > nav > ul > li:last-of-type > a",
result: {
title: "h2",
link: "a",
description: "p"
},
accept_terms_button: "#bnp_btn_accept"
}
};
module.exports = SearchScraper;