gotranslate-phrase-scrapper
Version:
GoTranslate Phrase Scrapper component for GoTranslate Framework, which scraps translatable phrases from remote websites.
236 lines (195 loc) • 6.57 kB
JavaScript
var request = require('request');
var cheerio = require('cheerio');
var async = require('async');
var Entities = require("html-entities").AllHtmlEntities;
var htmlEntities = new Entities();
var started = new Date(), ended, elapsed;
var contentType;
var pageData = {};
var $ = null;
if (typeof String.prototype.startsWith != 'function') {
String.prototype.startsWith = function (str){
return this.slice(0, str.length) == str;
};
}
if (typeof String.prototype.endsWith != 'function') {
String.prototype.endsWith = function (str){
return this.slice(-str.length) == str;
};
}
function isValidURL (url) {
var urlRegExp = /^(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:\/[^\s]*)?$/i;
return ((url != null) && (typeof url != 'undefined') && urlRegExp.test(url));
}
function Segment(segmentText, segmentMarkup, segmentElement) {
this.segmentElement = segmentElement.tagName;
this.segmentText = segmentText;
this.segmentTextLength = segmentText.length;
this.segmentMarkup = segmentMarkup;
this.segmentMarkupLength = segmentMarkup.length;
this.matches = (this.segmentText === this.segmentMarkup);
}
function createSegment(node) {
if (node == null) {
return null;
} else {
return new Segment($(node).text(), $(node).html(), node);
}
}
function getTextNodes (el) {
return $(el).find("*").contents().filter(function(){
var isCommentElement = this.nodeType == 8;
var isValidElement = ((this.nodeType == 1) && (excludedTags(this.tagName) == false));
var isValidText = ((this.nodeValue != null) && (this.nodeValue != 'undefined') && (this.nodeValue.trim().length > 0));
var isValidParentText = (this.nodeType == 3) && (this.parentNode != null) && (this.parentNode.tagName != 'undefined') && (excludedParents(this.parentNode.tagName) == false);
if (isCommentElement == true) {
return false;
} else if (isValidParentText == true) {
return false;
} else {
return isValidElement || isValidText;
}
}).filter(function(){
var isTextNodeAsDirectChild = false;
if ((this.childNodes != null) && (this.childNodes.length > 0)) {
$(this.childNodes).each(function(index){
if ((this != 'undefined') && (this.nodeType == 3) && (this.nodeValue.trim().length > 0)) {
isTextNodeAsDirectChild = true;
}
});
}
return isTextNodeAsDirectChild;
}).filter(function () {
var isSkippableNode = excludedTags(this.tagName) != 0;
var isSkippableParent = excludedParents(this.parentNode.tagName) != 0;
return !(isSkippableNode == false) || (isSkippableParent == false);
});
}
function excludedTags (nodeName) {
var found = false;
switch (nodeName) {
case "noscript":
case "table":
case "tbody":
case "thead":
case "tr":
case "ul":
case "ol":
case "script":
case "style":
case "code":
case "pre":
found = true;
break;
default:
found = false;
};
return found;
}
function excludedParents (nodeName) {
var found = false;
switch (nodeName) {
case "script":
case "style":
case "code":
case "pre":
found = true;
break;
default:
found = false;
};
return found;
}
function extractPhrases(url, root, jquery) {
var $ = jquery;
var segments = [];
$(getTextNodes($(root))).each(function(index, el){
segments.push(createSegment(this));
});
console.log("Total Segments = #" + segments.length);
return {
url: url,
count: segments.length,
phrases: segments,
htmlSource: "",
elapsedTime: ""
};
}
function scrapePhrases(url) {
if (url) {
if (isValidURL(url) === true) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
var requestOptions = {
method: 'GET'
, uri: url
, followRedirect : true
, followAllRedirects : true
};
var cheerioOptions = {
normalizeWhitespace: true
, xmlMode: false
, decodeEntities: true
, lowerCaseTags: true
};
console.log("%s - Scrapping of webpage located at %s is started.", started, url);
async.series ([
function (callback) {
request(requestOptions, function(error, response, html) {
if (error) {
return callback(error);
}
try {
contentType = response.headers['content-type'];
if ((contentType) && (contentType.trim().length > 0) && (contentType.startsWith("text/html"))) {
$ = cheerio.load(html, cheerioOptions);
callback();
} else {
throw new Error("Sorry! Unable to scrape phrases from '" + url + "' as it has unsupported content-type header i.e. " + contentType);
}
} catch (e) {
return callback(e);
}
});
},
function (callback) {
try {
pageData = extractPhrases(url, $.root(), $);
console.log("Total Phrases = #" + pageData.phrases.length);
if ((pageData) && (pageData.phrases)) {
callback();
} else {
throw new Error("Sorry! No translatable phrases found for '" + url + "'");
}
} catch (e) {
console.log("@3 - " + e);
return callback(e);
}
}
], function (err) {
if (err) {
console.log("@CALLBACK - " + err);
throw err;
}
ended = new Date();
elapsed = (ended - started) * 0.001 + " seconds";
console.log("%s - Scrapping of webpage located at %s is completed in %s", ended, url, elapsed);
return pageData;
});
} else {
throw new Error("Required 'url' parameter found null or has invalid value");
}
} else {
throw new Error("Required 'url' parameter found null or has invalid value");
}
console.log("Exiting");
}
exports.scrape = function (url, callback) {
try {
callback(null, scrapePhrases(url));
} catch (e) {
callback(e, null);
}
}
exports.metadata = function () {
console.log('This is it');
};