gotranslate-phrase-scrapper
Version:
GoTranslate Phrase Scrapper component for GoTranslate Framework, which scraps translatable phrases from remote websites.
318 lines (263 loc) • 9.68 kB
JavaScript
var request = require('request');
var cheerio = require('cheerio');
var Entities = require("html-entities").AllHtmlEntities;
var htmlEntities = new Entities();
var started = new Date(), ended, elapsed;
var pageData = {};
var $ = null;
var toolkit = {
name: require('./package').name,
version: require('./package').version
};
var requestDefaultOptions = {
method: 'GET'
, uri: ""
, followRedirect : true
, followAllRedirects : true
};
var cheerioDefaultOptions = {
normalizeWhitespace: true
, xmlMode: false //false
, decodeEntities: true
, lowerCaseTags: true
};
var scrapperDefaultOptions = {
includeHeaders: false
, includeMetadata: false
, includeFrames: false
, modifySource: false
};
if (typeof String.prototype.startsWith != 'function') {
String.prototype.startsWith = function (str){
return this.slice(0, str.length) == str;
};
}
if (typeof String.prototype.endsWith != 'function') {
String.prototype.endsWith = function (str){
return this.slice(-str.length) == str;
};
}
function isValidURL (url) {
var urlRegExp = /^(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:\/[^\s]*)?$/i;
return ((url != null) && (typeof url != 'undefined') && urlRegExp.test(url));
}
function Segment(segmentText, segmentMarkup, segmentElement) {
this.segmentElement = segmentElement.tagName;
this.segmentText = htmlEntities.decode(segmentText);
this.segmentTextLength = this.segmentText.length;
this.segmentMarkup = htmlEntities.decode(segmentMarkup);
this.segmentMarkupLength = this.segmentMarkup.length;
this.matches = (this.segmentText === this.segmentMarkup);
}
function createSegment(node) {
if (node == null) {
return null;
} else {
return new Segment($(node).text(), $(node).html(), node);
}
}
function getTextContents() {
var buffer = "";
$(pageData.phrases).each(function(index, phrase) {
if (buffer.indexOf(phrase['segmentText']) == -1) {
buffer += phrase['segmentText'].trim() + "\n";
}
});
return buffer;
}
function getTextNodes (el) {
return $(el).find("*").contents().filter(function(){
var isCommentElement = this.nodeType == 8;
var isValidElement = ((this.nodeType == 1) && (excludedTags(this.tagName) == false));
var isValidText = ((this.nodeValue != null) && (this.nodeValue != 'undefined') && (this.nodeValue.trim().length > 0));
var isValidParentText = (this.nodeType == 3) && (this.parentNode != null) && (this.parentNode.tagName != 'undefined') && (excludedParents(this.parentNode.tagName) == false);
if (isCommentElement == true) {
return false;
} else if (isValidParentText == true) {
return false;
} else {
return isValidElement || isValidText;
}
}).filter(function(){
var isTextNodeAsDirectChild = false;
if ((this.childNodes != null) && (this.childNodes.length > 0)) {
$(this.childNodes).each(function(index){
if ((this != 'undefined') && (this.nodeType == 3) && (this.nodeValue.trim().length > 0)) {
isTextNodeAsDirectChild = true;
}
});
}
return isTextNodeAsDirectChild;
}).filter(function () {
var isSkippableNode = excludedTags(this.tagName) != 0;
var isSkippableParent = excludedParents(this.parentNode.tagName) != 0;
return !(isSkippableNode == false) || (isSkippableParent == false);
});
}
function excludedTags (nodeName) {
var found = false;
switch (nodeName) {
case "noscript":
case "table":
case "tbody":
case "thead":
case "tr":
case "ul":
case "ol":
case "script":
case "style":
case "code":
case "pre":
found = true;
break;
default:
found = false;
};
return found;
}
function excludedParents (nodeName) {
var found = false;
switch (nodeName) {
case "script":
case "style":
case "code":
case "pre":
found = true;
break;
default:
found = false;
};
return found;
}
function extractPhrases(url, root, jquery) {
var $ = jquery;
var segments = [];
var segment;
$(getTextNodes($(root))).each(function(index, el){
segment = createSegment(this); //segments.push(createSegment(this));
if (scrapperDefaultOptions.modifySource) {
if (segment.segmentTextLength > 1) {
var dataPhrases = ((typeof $(this).parent().attr('data-phrases') === 'undefined') ? (index + 1) : $(this).parent().attr('data-phrases').concat(";").concat(index + 1));
$(this).parent().attr('data-phrases', dataPhrases);
$(this).attr('translate', "true");
$(this).attr('data-l10n-id', "gotps".concat(index + 1));
}
}
segment['translatable'] = ((segment.matches == true) && (segment.segmentTextLength > 1));
segment['segmentID'] = index + 1;
segments.push(segment);
});
return {
url: url,
count: segments.length,
phrases: segments
};
}
function getMetaInfo(metaTags) {
var metaInfo = [];
$(metaTags).each(function(index, meta){
metaInfo[index] = $(this).attr();
});
return metaInfo;
}
function getAnchors(anchorTags) {
var linkHref = [];
var href;
$(anchorTags).each(function(index, anchor){
href = $(this).attr('href');
if ((href != null) && (typeof href !== 'undefined') && (href.trim().length > 0)) {
if ((href.startsWith("#") || href.startsWith("javascript:")) === false) {
linkHref[linkHref.length] = href;
}
}
});
return linkHref;
}
function scrapePhrases(url, callback) {
if (url) {
if (isValidURL(url) === true) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
var requestOptions = requestDefaultOptions;
requestOptions.uri = url;
var cheerioOptions = cheerioDefaultOptions;
var contentType;
var validContentType = "text/html";
request(requestOptions, function(error, response, html) {
if (error) {
return callback(error, null);
}
try {
contentType = response.headers['content-type'];
if ((contentType) && (contentType.trim().length > 0) && (contentType.startsWith(validContentType))) {
$ = cheerio.load(html, cheerioOptions);
pageData = extractPhrases(url, $.root(), $);
if ((pageData) && (pageData.phrases)) {
// Hint: Modifying <title> of the page :-)
//$('title').text("GoTranslate Phrase Scrapper - " + $('title').text());
// Hint: Handling 'headers' property inclusion
if (scrapperDefaultOptions.includeHeaders) {
pageData['headers'] = response.headers;
}
// Hint: Adding 'htmlSource' property
pageData['sourceHTML'] = $.root().html();
pageData['sourceText'] = getTextContents(); //$('body').text();
// Hint: Handling 'frameSource' property inclusion
if (scrapperDefaultOptions.includeFrames) {
if ($('iframe').length > 0) {
pageData['frameSource'] = [$('iframe').length];
$('iframe').each(function(index, frame){
pageData['frameSource'][index] = $(frame).attr('src');
});
}
}
// Hint: Handling 'metaData' property inclusion
if (scrapperDefaultOptions.includeMetadata) {
if ($('meta').length > 0) {
pageData['metaData'] = getMetaInfo($('meta'));
}
}
// Hint: Adding 'whatLinks' property (if found)
if ($('a').length > 0) {
pageData['whatLinks'] = getAnchors($('a'));
}
ended = new Date();
elapsed = (ended - started) * 0.001 + "s";
pageData['startedOn'] = started;
pageData['endedOn'] = ended;
pageData['elapsedTime'] = elapsed;
pageData['toolkit'] = toolkit;
callback(null, pageData);
} else {
throw new Error("Sorry! No translatable phrases found for '" + url + "'");
}
} else {
throw new Error("Sorry! Unable to scrape phrases from '" + url + "' as it has unsupported content-type header i.e. " + contentType);
}
} catch (e) {
return callback(e, null);
}
});
} else {
return callback(new Error("Required 'url' parameter found null or has invalid value"), null);
}
} else {
return callback(new Error("Required 'url' parameter found null or has invalid value"), null);
}
}
exports.scrape = function (url, callback) {
return scrapePhrases(url, callback);
};
exports.setScrapperOptions = function (scrapperOptions) {
if ((typeof scrapperOptions !== 'object') || (scrapperOptions === {})) {
throw new Error("scrapperOptions must be a valid object");
}
try {
scrapperDefaultOptions.includeHeaders = scrapperOptions.includeHeaders || false;
scrapperDefaultOptions.includeMetadata = scrapperOptions.includeMetadata || false;
scrapperDefaultOptions.includeFrames = scrapperOptions.includeFrames || false;
scrapperDefaultOptions.modifySource = scrapperOptions.modifySource || false;
} catch (e) {
throw e;
}
};
exports.toolkit = toolkit;