hammer-scrape
Version:
Unifies Cheerio and Puppeteer for the most streamline scraping experience
305 lines • 12.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const request_group_puppeteer_1 = require("request-group-puppeteer");
const web_scraping_engine_1 = require("../web_scraping_engine");
const core_errors_1 = require("../core_errors");
/**
* Default configuration options for this parsing core
*/
exports.PUPPETEER_PARSING_CORE_DEFAULT = {
sharedRequest: null,
};
class PuppeteerParsingCore extends web_scraping_engine_1.ParsingCore {
constructor(url, sharedManager) {
super(url);
this.request = null;
this.initialized = false;
this.manager =
sharedManager !== undefined
? sharedManager
: new request_group_puppeteer_1.PuppeteerManager({
width: 1920,
height: 1080,
});
this.sharingManager = sharedManager !== undefined;
}
isInitialized() {
return this.initialized;
}
getRequest() {
if (this.request !== null) {
return this.request;
}
else {
throw new core_errors_1.CoreRequestNotCreatedError();
}
}
initialize(data = exports.PUPPETEER_PARSING_CORE_DEFAULT) {
return new Promise(async (resolve) => {
if (!this.isInitialized()) {
if (data.sharedRequest !== null) {
this.request = data.sharedRequest;
this.manager = data.sharedRequest.getManager();
}
// as of request-group-puppeteer version 1.2.0 there is no need to worry about reinitializing the puppeteer instance
await this.manager.initialize();
if (data.sharedRequest === null) {
this.request = new request_group_puppeteer_1.PuppeteerRequest(this.getUrl(), this.manager);
await this.request.run();
}
this.initialized = true;
}
resolve();
});
}
getText(querySelector) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let elementText = (await this.request
.getPage()
.evaluate(function (selector) {
return new Promise((evalResolve) => {
let resultText = '';
let elementResult = document.querySelector(selector);
if (elementResult) {
resultText = elementResult.textContent ? elementResult.textContent.trim() : '';
}
else {
resultText = '';
}
evalResolve(resultText);
});
}, querySelector));
resolve(elementText);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
getTextAll(querySelector) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let elementTexts = (await this.request
.getPage()
.evaluate(function (selector) {
return new Promise((evalResolve) => {
let resultText = [];
let elements = document.querySelectorAll(selector);
for (var i = 0; i < elements.length; i++) {
let element = elements[i];
if (element.textContent && element.textContent.trim().length > 0) {
resultText.push(element.textContent.trim());
}
}
evalResolve(resultText);
});
}, querySelector));
resolve(elementTexts);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
getAttribute(querySelector, attributeName) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let elementAttribute = (await this.request.getPage().evaluate(function (selector, selectedAttribute) {
return new Promise((evalResolve) => {
let element = document.querySelector(querySelector);
let attributeVal = element && element.getAttribute(selectedAttribute)
? element.getAttribute(selectedAttribute)
: '';
evalResolve(attributeVal);
});
}, querySelector, attributeName));
resolve(elementAttribute);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
getAttributeAll(querySelector, attributeName) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let elementAttributes = (await this.request.getPage().evaluate(function (selector, selectedAttribute) {
return new Promise((evalResolve) => {
let attributeValues = [];
let elements = document.querySelectorAll(querySelector);
for (var i = 0; i < elements.length; i++) {
let element = elements[i];
let attrVal = element.getAttribute(selectedAttribute)
? element.getAttribute(selectedAttribute).trim()
: '';
attributeValues.push(attrVal);
}
evalResolve(attributeValues);
});
}, querySelector, attributeName));
resolve(elementAttributes);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
getHtml(querySelector) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let htmlVal = (await this.request
.getPage()
.evaluate(function (selector) {
return new Promise(async (evalResolve) => {
let element = document.querySelector(selector);
let elementHtml = '';
if (element) {
elementHtml = element.outerHTML.trim();
}
evalResolve(elementHtml);
});
}, querySelector));
resolve(htmlVal);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
getHtmlAll(querySelector) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let htmlVal = (await this.request
.getPage()
.evaluate(function (selector) {
return new Promise(async (evalResolve) => {
let elementHtmls = [];
let elements = document.querySelectorAll(selector);
for (var i = 0; i < elements.length; i++) {
let element = elements[i];
elementHtmls.push(element.outerHTML);
}
evalResolve(elementHtmls);
});
}, querySelector));
resolve(htmlVal);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
elementExist(querySelector, timeout) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let elementExist = false;
try {
// attempt to wait for the element to pop up via selector.If the timeout exceeds, it fails
await this.request.getPage().waitForSelector(querySelector, {
timeout: timeout ? timeout : 10000,
});
elementExist = true;
}
catch (_a) {
// waiting for the selector failed. Element does not exist
elementExist = false;
}
resolve(elementExist);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
elementCount(querySelector) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let elementCounts = 0;
elementCounts = (await this.request
.getPage()
.evaluate(function (selector) {
return new Promise((evalResolve) => {
let totalElements = 0;
let elements = document.querySelectorAll(selector);
totalElements = elements.length;
evalResolve(totalElements);
});
}, querySelector));
resolve(elementCounts);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
getSelectOptions(querySelector) {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let selectOptions = [];
selectOptions = (await this.request
.getPage()
.evaluate(function (selector) {
return new Promise((evalResolve) => {
let options = [];
let selectElement = document.querySelector(selector);
if (selectElement) {
let optionElements = selectElement.querySelectorAll('option');
for (var i = 0; i < optionElements.length; i++) {
let option = optionElements[i];
let text = option.textContent ? option.textContent.trim() : '';
let val = option.getAttribute('value')
? option.getAttribute('value').trim()
: '';
// pushing parsed options
options.push({
text: text,
value: val,
});
}
}
evalResolve(options);
});
}, querySelector));
resolve(selectOptions);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
raw() {
if (this.isInitialized()) {
return this.request.getPage();
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
dispose() {
return new Promise(async (resolve) => {
if (this.request !== null) {
await this.request.dispose();
this.request = null;
}
if (this.manager !== null && !this.sharingManager) {
await this.manager.dispose();
}
resolve();
});
}
getDocumentHtml() {
if (this.isInitialized()) {
return new Promise(async (resolve) => {
let html = await this.raw().evaluate(() => document.body.outerHTML);
resolve(html);
});
}
else {
throw new core_errors_1.CoreNotInitializedError();
}
}
}
exports.PuppeteerParsingCore = PuppeteerParsingCore;
exports.default = PuppeteerParsingCore;
//# sourceMappingURL=puppeteer_parsing.js.map