@tacohitbox/bing-scraper
Version:
Scrape results from Microsoft Bing.
262 lines (245 loc) • 10.2 kB
JavaScript
const got = require("got");
const cheerio = require("cheerio");
const utils = require("./utils");
const cookieParser = require("set-cookie-parser");
exports.search = function(query, cb) {
var obj = {}
if (Object.prototype.toString.call(query) == "[object Object]") {
if (!query.q && !query.url) {
cb({
message: "No query or URL defined.",
code: "noQuery"
}, null);
return;
}
if (query.q) {obj.url = `https://www.bing.com/search?q=${encodeURIComponent(query.q)}`;} else {obj.url = query.url.toString();}
obj.userAgent = (query.userAgent || "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0");
obj.lang = (query.lang || "en-US"),
obj.referer = (query.referer || "https://www.bing.com/"),
obj.cookies = (query.cookies || undefined),
obj.pageCount = (query.pageCount || 1)
} else {
obj.url = `https://www.bing.com/search?q=${encodeURIComponent(query)}`;
obj.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0";
obj.lang = "en-US";
obj.referer = "https://www.bing.com/";
obj.cookies = undefined;
obj.pageCount = 1;
}
got(obj.url, {
headers: {
"Host": "www.bing.com",
"User-Agent": obj.userAgent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": obj.lang,
"Accept-Encoding": "gzip, deflate, br",
"Cookie": utils.cookieString(obj.cookies),
"Referer": obj.referer,
"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-GPC": "1",
"Cache-Control": "max-age=0",
"TE": "Trailers"
}
}).then(function(resp) {
var r = {};
var $ = cheerio.load(resp.body);
r.currentPage = resp.url;
if (
$(".sb_pagN")[0] !== undefined &&
$(".sb_pagN")[0].attribs !== undefined &&
$(".sb_pagN")[0].attribs.href !== undefined
) {
r.nextPage = `https://www.bing.com${$(".sb_pagN")[0].attribs.href}`;
} else {
r.nextPage = null;
}
r.cookies = cookieParser.parse(resp.headers["set-cookie"]);
r.results = [];
for (var c in $("#b_results .b_algo")) {
if (
$("#b_results .b_algo h2 a")[c] == undefined ||
$("#b_results .b_algo h2 a")[c].children == undefined ||
$("#b_results .b_algo h2 a")[c].children[0] == undefined ||
$("#b_results .b_algo h2 a")[c].children[0].data == undefined ||
$("#b_results .b_algo h2 a")[c].children[0].data == `!DOCTYPE html ""` ||
$("#b_results .b_algo .b_caption p")[c] == undefined ||
$("#b_results .b_algo .b_caption p")[c].children == undefined ||
$("#b_results .b_algo .b_caption p")[c].children[0] == undefined
) {continue;}
r.results.push({
"title": (utils.normalizeText($("#b_results .b_algo h2 a")[c].children) || ""),
"url": $("#b_results .b_algo h2 a")[c].attribs.href,
"description": (utils.normalizeText($("#b_results .b_algo .b_caption p")[c].children) || "")
});
}
if (
$("#b_results .b_ans .b_tppFact .b_imagePair")[0] !== undefined &&
$("#b_results .b_top .b_tppFact .b_imagePair .b_focusTextMedium")[0] !== undefined &&
$("#b_results .b_top .b_tppFact .b_imagePair .b_focusTextMedium")[0].children !== undefined &&
$("#b_results .b_top .b_tppFact .b_imagePair .b_focusLabel")[0] !== undefined &&
$("#b_results .b_top .b_tppFact .b_imagePair .b_focusLabel")[0].children !== undefined
) {
r.topAnswer = {
answer: utils.normalizeText($("#b_results .b_top .b_tppFact .b_imagePair .b_focusTextMedium")[0].children),
title: utils.normalizeText($("#b_results .b_top .b_tppFact .b_imagePair .b_focusLabel")[0].children),
image: (`https://www.bing.com${$("#b_results .b_top .b_tppFact .b_imagePair .cico img")[0].attribs["data-src-hq"]}` || null)
};
}
if (
$("#b_results .b_ans .qna-mf")[0] !== undefined &&
$("#b_results .b_ans .qna-mf .b_vPanel .rwrl")[0] !== undefined &&
$("#b_results .b_ans .qna-mf .b_vPanel .qna_algo a")[0] !== undefined &&
$("#b_results .b_ans .qna-mf .b_vPanel .qna_algo a")[0].children !== undefined &&
$("#b_results .b_ans .qna-mf .b_vPanel .qna_algo a")[0].children[0] !== undefined
) {
r.qnaAnswer = {
answer: utils.normalizeText($("#b_results .b_ans .qna-mf .b_vPanel .rwrl")[0].children),
source: {
title: utils.normalizeText($("#b_results .b_ans .qna-mf .b_vPanel .qna_algo a")[0].children),
url: $("#b_results .b_ans .qna-mf .b_vPanel .qna_algo a")[0].attribs.href
}
}
}
if (
$(".entityHeroVideo .b_videocard")[0] !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container")[0] !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container h3")[0] !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container h3")[0].children !== undefined &&
utils.normalizeText($(".entityHeroVideo .b_videocard .video_metadata_container h3")[0].children) !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container .video_summary")[0] !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container .video_summary")[0].children !== undefined &&
utils.normalizeText($(".entityHeroVideo .b_videocard .video_metadata_container .video_summary")[0].children) !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container .sav-ctrl__link")[0] !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container .sav-ctrl__link")[0].attribs !== undefined &&
$(".entityHeroVideo .b_videocard .video_metadata_container .sav-ctrl__link")[0].attribs["data-itemmeta"] !== undefined
) {
var d = JSON.parse($(".entityHeroVideo .b_videocard .video_metadata_container .sav-ctrl__link")[0].attribs["data-itemmeta"]);
r.videoObject = {
"title": utils.normalizeText($(".entityHeroVideo .b_videocard .video_metadata_container h3")[0].children),
"meta": utils.normalizeText($(".entityHeroVideo .b_videocard .video_metadata_container .video_summary")[0].children),
"url": (d.videos[0].murl || d.videos[0].purl || null)
};
}
var pc = (obj.pageCount - 1);
if (pc > 0 && r.nextPage !== null) {
require(".").search({
url: r.nextPage,
pageCount: pc,
userAgent: obj.userAgent,
cookies: r.cookies
}, function(err, resp) {
if (err) {cb(err, null);} else {
r.currentPage = r.nextPage;
r.nextPage = resp.nextPage;
for (var c in resp.results) {
r.results.push(resp.results[c]);
}
cb(null, r);
}
})
} else {
cb(null, r);
}
}).catch(function(err) {
cb(err, null);
});
}
exports.searchImages = function(query, cb) {
var obj = {}
if (Object.prototype.toString.call(query) == "[object Object]") {
if (!query.q && !query.url) {
cb({
message: "No query or URL defined.",
code: "noQuery"
}, null);
return;
}
if (query.q) {obj.url = `https://www.bing.com/images/search?q=${encodeURIComponent(query.q)}`;} else {obj.url = query.url.toString();}
if (query.enforceLanguage && !obj.url.includes("lf=1")) {obj.url = `${obj.url}&lf=1`;}
obj.userAgent = (query.userAgent || "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0");
obj.lang = (query.lang || "en-US"),
obj.referer = (query.referer || "https://www.bing.com/"),
obj.cookies = (query.cookies || undefined),
obj.pageCount = (query.pageCount || 1)
} else {
obj.url = `https://www.bing.com/images/search?q=${encodeURIComponent(query)}`;
obj.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0";
obj.lang = "en-US";
obj.referer = "https://www.bing.com/";
obj.cookies = undefined;
obj.pageCount = 1;
}
got(obj.url, {
headers: {
"Host": "www.bing.com",
"User-Agent": obj.userAgent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": obj.lang,
"Accept-Encoding": "gzip, deflate, br",
"Cookie": utils.cookieString(obj.cookies),
"Referer": obj.referer,
"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-GPC": "1",
"Cache-Control": "max-age=0",
"TE": "Trailers"
}
}).then(function(resp) {
var r = {};
var $ = cheerio.load(resp.body);
r.currentPage = resp.url;
if (
$(".dg_b .dgControl")[0] !== undefined &&
$(".dg_b .dgControl")[0].attribs !== undefined &&
$(".dg_b .dgControl")[0].attribs["data-nexturl"] !== undefined
) {
r.nextPage = `https://www.bing.com${$(".dg_b .dgControl")[0].attribs["data-nexturl"]}`;
} else {
r.nextPage = null;
}
r.cookies = cookieParser.parse(resp.headers["set-cookie"]);
r.results = [];
for (var c in $(".dg_b .iusc")) {
if ($(".dg_b .iusc")[c] !== undefined && $(".dg_b .iusc")[c].attribs !== undefined) {
if ($(".dg_b .iusc")[c].attribs["m"]) {
var j = JSON.parse($(".dg_b .iusc")[c].attribs["m"]);
if (j == null) {continue;}
r.results.push({
"thumbnail": j.turl,
"source": j.purl,
"direct": j.murl,
"description": j.desc,
"title": j.t
});
} else {
continue;
}
} else {
continue;
}
}
var pc = (obj.pageCount - 1);
if (pc > 0 && r.nextPage !== null) {
require(".").searchImages({
url: r.nextPage,
pageCount: pc,
userAgent: obj.userAgent,
cookies: r.cookies
}, function(err, resp) {
if (err) {cb(err, null);} else {
for (var c in resp.results) {
r.results.push(resp.results[c]);
}
cb(null, r);
}
})
} else {
cb(null, r);
}
}).catch(function(err) {
cb(err, null);
});
}