char-encoding-detector
Version:
Character encoding detector
1,179 lines (983 loc) • 76.1 kB
JavaScript
function _classCallCheck(instance, Constructor) {
if (!(instance instanceof Constructor)) {
throw new TypeError("Cannot call a class as a function");
}
}
function _defineProperties(target, props) {
for (var i = 0; i < props.length; i++) {
var descriptor = props[i];
descriptor.enumerable = descriptor.enumerable || false;
descriptor.configurable = true;
if ("value" in descriptor) descriptor.writable = true;
Object.defineProperty(target, descriptor.key, descriptor);
}
}
function _createClass(Constructor, protoProps, staticProps) {
if (protoProps) _defineProperties(Constructor.prototype, protoProps);
if (staticProps) _defineProperties(Constructor, staticProps);
return Constructor;
}
function _inherits(subClass, superClass) {
if (typeof superClass !== "function" && superClass !== null) {
throw new TypeError("Super expression must either be null or a function");
}
subClass.prototype = Object.create(superClass && superClass.prototype, {
constructor: {
value: subClass,
writable: true,
configurable: true
}
});
if (superClass) _setPrototypeOf(subClass, superClass);
}
function _getPrototypeOf(o) {
_getPrototypeOf = Object.setPrototypeOf ? Object.getPrototypeOf : function _getPrototypeOf(o) {
return o.__proto__ || Object.getPrototypeOf(o);
};
return _getPrototypeOf(o);
}
function _setPrototypeOf(o, p) {
_setPrototypeOf = Object.setPrototypeOf || function _setPrototypeOf(o, p) {
o.__proto__ = p;
return o;
};
return _setPrototypeOf(o, p);
}
function _assertThisInitialized(self) {
if (self === void 0) {
throw new ReferenceError("this hasn't been initialised - super() hasn't been called");
}
return self;
}
function _possibleConstructorReturn(self, call) {
if (call && (typeof call === "object" || typeof call === "function")) {
return call;
}
return _assertThisInitialized(self);
}
var Match = function Match(det, rec, confidence, name, lang) {
_classCallCheck(this, Match);
this.confidence = confidence;
this.name = name || rec.name(det);
this.lang = lang;
};
/**
* Charset recognizer for UTF-8
*/
var utf8 =
/*#__PURE__*/
function () {
function utf8() {
_classCallCheck(this, utf8);
}
_createClass(utf8, [{
key: "name",
value: function name() {
return 'UTF-8';
}
}, {
key: "match",
value: function match(det) {
var hasBOM = false,
numValid = 0,
numInvalid = 0,
input = det.fRawInput,
trailBytes = 0,
confidence;
if (det.fRawLength >= 3 && (input[0] & 0xff) == 0xef && (input[1] & 0xff) == 0xbb && (input[2] & 0xff) == 0xbf) {
hasBOM = true;
} // Scan for multi-byte sequences
for (var i = 0; i < det.fRawLength; i++) {
var b = input[i];
if ((b & 0x80) == 0) continue; // ASCII
// Hi bit on char found. Figure out how long the sequence should be
if ((b & 0x0e0) == 0x0c0) {
trailBytes = 1;
} else if ((b & 0x0f0) == 0x0e0) {
trailBytes = 2;
} else if ((b & 0x0f8) == 0xf0) {
trailBytes = 3;
} else {
numInvalid++;
if (numInvalid > 5) break;
trailBytes = 0;
} // Verify that we've got the right number of trail bytes in the sequence
for (;;) {
i++;
if (i >= det.fRawLength) break;
if ((input[i] & 0xc0) != 0x080) {
numInvalid++;
break;
}
if (--trailBytes == 0) {
numValid++;
break;
}
}
} // Cook up some sort of confidence score, based on presense of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
confidence = 0;
if (hasBOM && numInvalid == 0) confidence = 100;else if (hasBOM && numValid > numInvalid * 10) confidence = 80;else if (numValid > 3 && numInvalid == 0) confidence = 100;else if (numValid > 0 && numInvalid == 0) confidence = 80;else if (numValid == 0 && numInvalid == 0) // Plain ASCII.
confidence = 10;else if (numValid > numInvalid * 10) // Probably corruput utf-8 data. Valid sequences aren't likely by chance.
confidence = 25;else return null;
return new Match(det, this, confidence);
}
}]);
return utf8;
}();
/**
* Binary search implementation (recursive)
*/
function binarySearch(arr, searchValue) {
function find(arr, searchValue, left, right) {
if (right < left) return -1;
/*
int mid = mid = (left + right) / 2;
There is a bug in the above line;
Joshua Bloch suggests the following replacement:
*/
var mid = Math.floor(left + right >>> 1);
if (searchValue > arr[mid]) return find(arr, searchValue, mid + 1, right);
if (searchValue < arr[mid]) return find(arr, searchValue, left, mid - 1);
return mid;
}
return find(arr, searchValue, 0, arr.length - 1);
} // 'Character' iterated character class.
// Recognizers for specific mbcs encodings make their 'characters' available
// by providing a nextChar() function that fills in an instance of iteratedChar
// with the next char from the input.
// The returned characters are not converted to Unicode, but remain as the raw
// bytes (concatenated into an int) from the codepage data.
//
// For Asian charsets, use the raw input rather than the input that has been
// stripped of markup. Detection only considers multi-byte chars, effectively
// stripping markup anyway, and double byte chars do occur in markup too.
//
var IteratedChar =
/*#__PURE__*/
function () {
function IteratedChar() {
_classCallCheck(this, IteratedChar);
this.charValue = 0; // 1-4 bytes from the raw input data
this.index = 0;
this.nextIndex = 0;
this.error = false;
this.done = false;
}
_createClass(IteratedChar, [{
key: "reset",
value: function reset() {
this.charValue = 0;
this.index = -1;
this.nextIndex = 0;
this.error = false;
this.done = false;
}
}, {
key: "nextByte",
value: function nextByte(det) {
if (this.nextIndex >= det.fRawLength) {
this.done = true;
return -1;
}
var byteValue = det.fRawInput[this.nextIndex++] & 0x00ff;
return byteValue;
}
}]);
return IteratedChar;
}();
/**
* Asian double or multi-byte - charsets.
* Match is determined mostly by the input data adhering to the
* encoding scheme for the charset, and, optionally,
* frequency-of-occurence of characters.
*/
var mbcs =
/*#__PURE__*/
function () {
function mbcs() {
_classCallCheck(this, mbcs);
}
_createClass(mbcs, [{
key: "name",
value: function name() {
return '[abstract]mbcs';
}
/**
* Test the match of this charset with the input text data
* which is obtained via the CharsetDetector object.
*
* @param det The CharsetDetector, which contains the input text
* to be checked for being in this charset.
* @return Two values packed into one int (Damn java, anyhow)
* bits 0-7: the match confidence, ranging from 0-100
* bits 8-15: The match reason, an enum-like value.
*/
}, {
key: "match",
value: function match(det) {
var doubleByteCharCount = 0,
commonCharCount = 0,
badCharCount = 0,
totalCharCount = 0,
confidence = 0;
var iter = new IteratedChar();
detectBlock: {
for (iter.reset(); this.nextChar(iter, det);) {
totalCharCount++;
if (iter.error) {
badCharCount++;
} else {
var cv = iter.charValue & 0xffffffff;
if (cv <= 0xff) ; else {
doubleByteCharCount++;
if (this.commonChars != null) {
// NOTE: This assumes that there are no 4-byte common chars.
if (binarySearch(this.commonChars, cv) >= 0) {
commonCharCount++;
}
}
}
}
if (badCharCount >= 2 && badCharCount * 5 >= doubleByteCharCount) {
// console.log('its here!')
// Bail out early if the byte data is not matching the encoding scheme.
break detectBlock;
}
}
if (doubleByteCharCount <= 10 && badCharCount == 0) {
// Not many multi-byte chars.
if (doubleByteCharCount == 0 && totalCharCount < 10) {
// There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes.
// We don't have enough data to have any confidence.
// Statistical analysis of single byte non-ASCII charcters would probably help here.
confidence = 0;
} else {
// ASCII or ISO file? It's probably not our encoding,
// but is not incompatible with our encoding, so don't give it a zero.
confidence = 10;
}
break detectBlock;
} //
// No match if there are too many characters that don't fit the encoding scheme.
// (should we have zero tolerance for these?)
//
if (doubleByteCharCount < 20 * badCharCount) {
confidence = 0;
break detectBlock;
}
if (this.commonChars == null) {
// We have no statistics on frequently occuring characters.
// Assess confidence purely on having a reasonable number of
// multi-byte characters (the more the better
confidence = 30 + doubleByteCharCount - 20 * badCharCount;
if (confidence > 100) {
confidence = 100;
}
} else {
//
// Frequency of occurence statistics exist.
//
var maxVal = Math.log(parseFloat(doubleByteCharCount) / 4);
var scaleFactor = 90.0 / maxVal;
confidence = Math.floor(Math.log(commonCharCount + 1) * scaleFactor + 10);
confidence = Math.min(confidence, 100);
}
} // end of detectBlock:
return confidence == 0 ? null : new Match(det, this, confidence);
}
/**
* Get the next character (however many bytes it is) from the input data
* Subclasses for specific charset encodings must implement this function
* to get characters according to the rules of their encoding scheme.
*
* This function is not a method of class iteratedChar only because
* that would require a lot of extra derived classes, which is awkward.
* @param it The iteratedChar 'struct' into which the returned char is placed.
* @param det The charset detector, which is needed to get at the input byte data
* being iterated over.
* @return True if a character was returned, false at end of input.
*/
}, {
key: "nextChar",
value: function nextChar(iter, det) {
return false;
}
}]);
return mbcs;
}();
/**
* Shift-JIS charset recognizer.
*/
var sjis =
/*#__PURE__*/
function (_mbcs) {
_inherits(sjis, _mbcs);
function sjis() {
var _this;
_classCallCheck(this, sjis);
_this = _possibleConstructorReturn(this, _getPrototypeOf(sjis).apply(this, arguments)); // TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
_this.commonChars = [0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0, 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3, 0x82b5, 0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8, 0x82c9, 0x82cc, 0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea, 0x82f0, 0x82f1, 0x8341, 0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389, 0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa];
return _this;
}
_createClass(sjis, [{
key: "name",
value: function name() {
return 'Shift-JIS';
}
}, {
key: "language",
value: function language() {
return 'ja';
}
}, {
key: "nextChar",
value: function nextChar(iter, det) {
iter.index = iter.nextIndex;
iter.error = false;
var firstByte;
firstByte = iter.charValue = iter.nextByte(det);
if (firstByte < 0) return false;
if (firstByte <= 0x7f || firstByte > 0xa0 && firstByte <= 0xdf) return true;
var secondByte = iter.nextByte(det);
if (secondByte < 0) return false;
iter.charValue = firstByte << 8 | secondByte;
if (!(secondByte >= 0x40 && secondByte <= 0x7f || secondByte >= 0x80 && secondByte <= 0xff)) {
// Illegal second byte value.
iter.error = true;
}
return true;
}
}]);
return sjis;
}(mbcs);
/**
* Big5 charset recognizer.
*/
var big5 =
/*#__PURE__*/
function (_mbcs2) {
_inherits(big5, _mbcs2);
function big5() {
var _this2;
_classCallCheck(this, big5);
_this2 = _possibleConstructorReturn(this, _getPrototypeOf(big5).apply(this, arguments)); // TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
_this2.commonChars = [0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446, 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477, 0xa4a3, 0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd, 0xa540, 0xa548, 0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662, 0xa668, 0xa670, 0xa6a8, 0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da, 0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, 0xaa6b, 0xaaba, 0xaabe, 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, 0xaec9, 0xafe0, 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, 0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, 0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f];
return _this2;
}
_createClass(big5, [{
key: "name",
value: function name() {
return 'Big5';
}
}, {
key: "language",
value: function language() {
return 'zh';
}
}, {
key: "nextChar",
value: function nextChar(iter, det) {
iter.index = iter.nextIndex;
iter.error = false;
var firstByte = iter.charValue = iter.nextByte(det);
if (firstByte < 0) return false; // single byte character.
if (firstByte <= 0x7f || firstByte == 0xff) return true;
var secondByte = iter.nextByte(det);
if (secondByte < 0) return false;
iter.charValue = iter.charValue << 8 | secondByte;
if (secondByte < 0x40 || secondByte == 0x7f || secondByte == 0xff) iter.error = true;
return true;
}
}]);
return big5;
}(mbcs);
/**
* EUC charset recognizers. One abstract class that provides the common function
* for getting the next character according to the EUC encoding scheme,
* and nested derived classes for EUC_KR, EUC_JP, EUC_CN.
*
* Get the next character value for EUC based encodings.
* Character 'value' is simply the raw bytes that make up the character
* packed into an int.
*/
function eucNextChar(iter, det) {
iter.index = iter.nextIndex;
iter.error = false;
var firstByte = 0;
var secondByte = 0;
var thirdByte = 0; //int fourthByte = 0;
buildChar: {
firstByte = iter.charValue = iter.nextByte(det);
if (firstByte < 0) {
// Ran off the end of the input data
iter.done = true;
break buildChar;
}
if (firstByte <= 0x8d) {
// single byte char
break buildChar;
}
secondByte = iter.nextByte(det);
iter.charValue = iter.charValue << 8 | secondByte;
if (firstByte >= 0xa1 && firstByte <= 0xfe) {
// Two byte Char
if (secondByte < 0xa1) {
iter.error = true;
}
break buildChar;
}
if (firstByte == 0x8e) {
// Code Set 2.
// In EUC-JP, total char size is 2 bytes, only one byte of actual char value.
// In EUC-TW, total char size is 4 bytes, three bytes contribute to char value.
// We don't know which we've got.
// Treat it like EUC-JP. If the data really was EUC-TW, the following two
// bytes will look like a well formed 2 byte char.
if (secondByte < 0xa1) {
iter.error = true;
}
break buildChar;
}
if (firstByte == 0x8f) {
// Code set 3.
// Three byte total char size, two bytes of actual char value.
thirdByte = iter.nextByte(det);
iter.charValue = iter.charValue << 8 | thirdByte;
if (thirdByte < 0xa1) {
iter.error = true;
}
}
}
return iter.done == false;
}
/**
* The charset recognize for EUC-JP. A singleton instance of this class
* is created and kept by the public CharsetDetector class
*/
var euc_jp =
/*#__PURE__*/
function (_mbcs3) {
_inherits(euc_jp, _mbcs3);
function euc_jp() {
var _this3;
_classCallCheck(this, euc_jp);
_this3 = _possibleConstructorReturn(this, _getPrototypeOf(euc_jp).apply(this, arguments)); // TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
_this3.commonChars = [0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2, 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1, 0xa4b3, 0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1, 0xa4c3, 0xa4c4, 0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de, 0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, 0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, 0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, 0xa5c8, 0xa5c9, 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, 0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, 0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1];
return _this3;
}
_createClass(euc_jp, [{
key: "name",
value: function name() {
return 'EUC-JP';
}
}, {
key: "language",
value: function language() {
return 'ja';
}
}, {
key: "nextChar",
value: function nextChar(iter, det) {
return eucNextChar(iter, det);
}
}]);
return euc_jp;
}(mbcs);
/**
* The charset recognize for EUC-KR. A singleton instance of this class
* is created and kept by the public CharsetDetector class
*/
var euc_kr =
/*#__PURE__*/
function (_mbcs4) {
_inherits(euc_kr, _mbcs4);
function euc_kr() {
var _this4;
_classCallCheck(this, euc_kr);
_this4 = _possibleConstructorReturn(this, _getPrototypeOf(euc_kr).apply(this, arguments)); // TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
_this4.commonChars = [0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc, 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf, 0xb4d9, 0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af, 0xb7c2, 0xb7ce, 0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce, 0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, 0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, 0xbef8, 0xbefa, 0xbfa1, 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, 0xc0af, 0xc0b8, 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, 0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, 0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad];
return _this4;
}
_createClass(euc_kr, [{
key: "name",
value: function name() {
return 'EUC-KR';
}
}, {
key: "language",
value: function language() {
return 'ko';
}
}, {
key: "nextChar",
value: function nextChar(iter, det) {
return eucNextChar(iter, det);
}
}]);
return euc_kr;
}(mbcs);
/**
* GB-18030 recognizer. Uses simplified Chinese statistics.
*/
var gb_18030 =
/*#__PURE__*/
function (_mbcs5) {
_inherits(gb_18030, _mbcs5);
function gb_18030() {
var _this5;
_classCallCheck(this, gb_18030);
_this5 = _possibleConstructorReturn(this, _getPrototypeOf(gb_18030).apply(this, arguments)); // TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
_this5.commonChars = [0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac, 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd, 0xb5c4, 0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6, 0xb7dd, 0xb8b4, 0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd, 0xbacd, 0xbba7, 0xbbd6, 0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, 0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, 0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, 0xcad0, 0xcad6, 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, 0xcfb5, 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, 0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0];
return _this5;
}
_createClass(gb_18030, [{
key: "name",
value: function name() {
return 'GB18030';
}
}, {
key: "language",
value: function language() {
return 'zh';
}
/*
* Get the next character value for EUC based encodings.
* Character 'value' is simply the raw bytes that make up the character
* packed into an int.
*/
}, {
key: "nextChar",
value: function nextChar(iter, det) {
iter.index = iter.nextIndex;
iter.error = false;
var firstByte = 0;
var secondByte = 0;
var thirdByte = 0;
var fourthByte = 0;
buildChar: {
firstByte = iter.charValue = iter.nextByte(det);
if (firstByte < 0) {
// Ran off the end of the input data
iter.done = true;
break buildChar;
}
if (firstByte <= 0x80) {
// single byte char
break buildChar;
}
secondByte = iter.nextByte(det);
iter.charValue = iter.charValue << 8 | secondByte;
if (firstByte >= 0x81 && firstByte <= 0xfe) {
// Two byte Char
if (secondByte >= 0x40 && secondByte <= 0x7e || secondByte >= 80 && secondByte <= 0xfe) {
break buildChar;
} // Four byte char
if (secondByte >= 0x30 && secondByte <= 0x39) {
thirdByte = iter.nextByte(det);
if (thirdByte >= 0x81 && thirdByte <= 0xfe) {
fourthByte = iter.nextByte(det);
if (fourthByte >= 0x30 && fourthByte <= 0x39) {
iter.charValue = iter.charValue << 16 | thirdByte << 8 | fourthByte;
break buildChar;
}
}
}
iter.error = true;
break buildChar;
}
}
return iter.done == false;
}
}]);
return gb_18030;
}(mbcs);
/**
* This class matches UTF-16 and UTF-32, both big- and little-endian. The
* BOM will be used if it is present.
*/
var UTF_16BE =
/*#__PURE__*/
function () {
function UTF_16BE() {
_classCallCheck(this, UTF_16BE);
}
_createClass(UTF_16BE, [{
key: "name",
value: function name() {
return 'UTF-16BE';
}
}, {
key: "match",
value: function match(det) {
var input = det.fRawInput;
if (input.length >= 2 && (input[0] & 0xff) == 0xfe && (input[1] & 0xff) == 0xff) {
return new Match(det, this, 100); // confidence = 100
} // TODO: Do some statistics to check for unsigned UTF-16BE
return null;
}
}]);
return UTF_16BE;
}();
var UTF_16LE =
/*#__PURE__*/
function () {
function UTF_16LE() {
_classCallCheck(this, UTF_16LE);
}
_createClass(UTF_16LE, [{
key: "name",
value: function name() {
return 'UTF-16LE';
}
}, {
key: "match",
value: function match(det) {
var input = det.fRawInput;
if (input.length >= 2 && (input[0] & 0xff) == 0xff && (input[1] & 0xff) == 0xfe) {
// LE BOM is present.
if (input.length >= 4 && input[2] == 0x00 && input[3] == 0x00) {
// It is probably UTF-32 LE, not UTF-16
return null;
}
return new Match(det, this, 100); // confidence = 100
} // TODO: Do some statistics to check for unsigned UTF-16LE
return null;
}
}]);
return UTF_16LE;
}();
var UTF_32 =
/*#__PURE__*/
function () {
function UTF_32() {
_classCallCheck(this, UTF_32);
}
_createClass(UTF_32, [{
key: "name",
value: function name() {
return '[abstract] UTF_32';
} // abstract
}, {
key: "getChar",
value: function getChar(input, number) {
return 0;
}
}, {
key: "match",
value: function match(det) {
var input = det.fRawInput,
limit = det.fRawLength / 4 * 4,
numValid = 0,
numInvalid = 0,
hasBOM = false,
confidence = 0;
if (limit == 0) {
return null;
}
if (this.getChar(input, 0) == 0x0000feff) {
hasBOM = true;
}
for (var i = 0; i < limit; i += 4) {
var ch = this.getChar(input, i);
if (ch < 0 || ch >= 0x10ffff || ch >= 0xd800 && ch <= 0xdfff) {
numInvalid += 1;
} else {
numValid += 1;
}
} // Cook up some sort of confidence score, based on presence of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
if (hasBOM && numInvalid == 0) {
confidence = 100;
} else if (hasBOM && numValid > numInvalid * 10) {
confidence = 80;
} else if (numValid > 3 && numInvalid == 0) {
confidence = 100;
} else if (numValid > 0 && numInvalid == 0) {
confidence = 80;
} else if (numValid > numInvalid * 10) {
// Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance.
confidence = 25;
} // return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
return confidence == 0 ? null : new Match(det, this, confidence);
}
}]);
return UTF_32;
}();
var UTF_32BE =
/*#__PURE__*/
function (_UTF_) {
_inherits(UTF_32BE, _UTF_);
function UTF_32BE() {
_classCallCheck(this, UTF_32BE);
return _possibleConstructorReturn(this, _getPrototypeOf(UTF_32BE).apply(this, arguments));
}
_createClass(UTF_32BE, [{
key: "name",
value: function name() {
return 'UTF-32BE';
}
}, {
key: "getChar",
value: function getChar(input, index) {
return (input[index + 0] & 0xff) << 24 | (input[index + 1] & 0xff) << 16 | (input[index + 2] & 0xff) << 8 | input[index + 3] & 0xff;
}
}]);
return UTF_32BE;
}(UTF_32);
var UTF_32LE =
/*#__PURE__*/
function (_UTF_2) {
_inherits(UTF_32LE, _UTF_2);
function UTF_32LE() {
_classCallCheck(this, UTF_32LE);
return _possibleConstructorReturn(this, _getPrototypeOf(UTF_32LE).apply(this, arguments));
}
_createClass(UTF_32LE, [{
key: "name",
value: function name() {
return 'UTF-32LE';
}
}, {
key: "getChar",
value: function getChar(input, index) {
return (input[index + 3] & 0xff) << 24 | (input[index + 2] & 0xff) << 16 | (input[index + 1] & 0xff) << 8 | input[index + 0] & 0xff;
}
}]);
return UTF_32LE;
}(UTF_32);
var N_GRAM_MASK = 0xffffff;
/**
* This class recognizes single-byte encodings. Because the encoding scheme is so
* simple, language statistics are used to do the matching.
*/
var NGramParser =
/*#__PURE__*/
function () {
function NGramParser(theNgramList, theByteMap) {
_classCallCheck(this, NGramParser);
this.byteIndex = 0;
this.ngram = 0;
this.ngramCount = 0;
this.hitCount = 0;
this.ngramList = theNgramList;
this.byteMap = theByteMap;
}
/*
* Binary search for value in table, which must have exactly 64 entries.
*/
_createClass(NGramParser, [{
key: "search",
value: function search(table, value) {
var index = 0;
if (table[index + 32] <= value) index += 32;
if (table[index + 16] <= value) index += 16;
if (table[index + 8] <= value) index += 8;
if (table[index + 4] <= value) index += 4;
if (table[index + 2] <= value) index += 2;
if (table[index + 1] <= value) index += 1;
if (table[index] > value) index -= 1;
if (index < 0 || table[index] != value) return -1;
return index;
}
}, {
key: "lookup",
value: function lookup(thisNgram) {
this.ngramCount += 1;
if (this.search(this.ngramList, thisNgram) >= 0) {
this.hitCount += 1;
}
}
}, {
key: "addByte",
value: function addByte(b) {
this.ngram = (this.ngram << 8) + (b & 0xff) & N_GRAM_MASK;
this.lookup(this.ngram);
}
}, {
key: "nextByte",
value: function nextByte(det) {
if (this.byteIndex >= det.fInputLen) return -1;
return det.fInputBytes[this.byteIndex++] & 0xff;
}
}, {
key: "parse",
value: function parse(det, spaceCh) {
var b,
ignoreSpace = false;
this.spaceChar = spaceCh;
while ((b = this.nextByte(det)) >= 0) {
var mb = this.byteMap[b]; // TODO: 0x20 might not be a space in all character sets...
if (mb != 0) {
if (!(mb == this.spaceChar && ignoreSpace)) {
this.addByte(mb);
}
ignoreSpace = mb == this.spaceChar;
}
} // TODO: Is this OK? The buffer could have ended in the middle of a word...
this.addByte(this.spaceChar);
var rawPercent = this.hitCount / this.ngramCount; // TODO - This is a bit of a hack to take care of a case
// were we were getting a confidence of 135...
if (rawPercent > 0.33) return 98;
return Math.floor(rawPercent * 300.0);
}
}]);
return NGramParser;
}();
var NGramsPlusLang = function NGramsPlusLang(la, ng) {
_classCallCheck(this, NGramsPlusLang);
this.fLang = la;
this.fNGrams = ng;
};
var sbcs =
/*#__PURE__*/
function () {
function sbcs() {
_classCallCheck(this, sbcs);
this.spaceChar = 0x20;
}
_createClass(sbcs, [{
key: "ngrams",
value: function ngrams() {
return [];
}
}, {
key: "byteMap",
value: function byteMap() {}
}, {
key: "name",
value: function name(det) {
return '[abstract] sbcs';
}
}, {
key: "match",
value: function match(det) {
var ngrams = this.ngrams();
var multiple = Array.isArray(ngrams) && ngrams[0] instanceof NGramsPlusLang;
if (!multiple) {
var parser = new NGramParser(ngrams, this.byteMap());
var confidence = parser.parse(det, this.spaceChar);
return confidence <= 0 ? null : new Match(det, this, confidence);
}
var bestConfidenceSoFar = -1;
var lang = null;
for (var i = ngrams.length - 1; i >= 0; i--) {
var ngl = ngrams[i];
var parser = new NGramParser(ngl.fNGrams, this.byteMap());
var confidence = parser.parse(det, this.spaceChar);
if (confidence > bestConfidenceSoFar) {
bestConfidenceSoFar = confidence;
lang = ngl.fLang;
}
}
var name = this.name(det);
return bestConfidenceSoFar <= 0 ? null : new Match(det, this, bestConfidenceSoFar, name, lang);
}
}]);
return sbcs;
}();
var ISO_8859_1 =
/*#__PURE__*/
function (_sbcs) {
_inherits(ISO_8859_1, _sbcs);
function ISO_8859_1() {
_classCallCheck(this, ISO_8859_1);
return _possibleConstructorReturn(this, _getPrototypeOf(ISO_8859_1).apply(this, arguments));
}
_createClass(ISO_8859_1, [{
key: "byteMap",
value: function byteMap() {
return [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xaa, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xb5, 0x20, 0x20, 0x20, 0x20, 0xba, 0x20, 0x20, 0x20, 0x20, 0x20, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff];
}
}, {
key: "ngrams",
value: function ngrams() {
return [new NGramsPlusLang('da', [0x206166, 0x206174, 0x206465, 0x20656e, 0x206572, 0x20666f, 0x206861, 0x206920, 0x206d65, 0x206f67, 0x2070e5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620, 0x616e20, 0x616e64, 0x617220, 0x617420, 0x646520, 0x64656e, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656e20, 0x656e64, 0x657220, 0x657265, 0x657320, 0x657420, 0x666f72, 0x676520, 0x67656e, 0x676572, 0x696765, 0x696c20, 0x696e67, 0x6b6520, 0x6b6b65, 0x6c6572, 0x6c6967, 0x6c6c65, 0x6d6564, 0x6e6465, 0x6e6520, 0x6e6720, 0x6e6765, 0x6f6720, 0x6f6d20, 0x6f7220, 0x70e520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696c, 0x766572]), new NGramsPlusLang('de', [0x20616e, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696e, 0x206d69, 0x207363, 0x207365, 0x20756e, 0x207665, 0x20766f, 0x207765, 0x207a75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656e, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696e, 0x656974, 0x656e20, 0x657220, 0x657320, 0x67656e, 0x68656e, 0x687420, 0x696368, 0x696520, 0x696e20, 0x696e65, 0x697420, 0x6c6963, 0x6c6c65, 0x6e2061, 0x6e2064, 0x6e2073, 0x6e6420, 0x6e6465, 0x6e6520, 0x6e6720, 0x6e6765, 0x6e7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656e, 0x746572, 0x756e64, 0x756e67, 0x766572]), new NGramsPlusLang('en', [0x206120, 0x20616e, 0x206265, 0x20636f, 0x20666f, 0x206861, 0x206865, 0x20696e, 0x206d61, 0x206f66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746f, 0x207768, 0x616964, 0x616c20, 0x616e20, 0x616e64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656e74, 0x657220, 0x657320, 0x666f72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696e20, 0x696e67, 0x696f6e, 0x697320, 0x6e2061, 0x6e2074, 0x6e6420, 0x6e6720, 0x6e7420, 0x6f6620, 0x6f6e20, 0x6f7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696f, 0x746f20, 0x747320]), new NGramsPlusLang('es', [0x206120, 0x206361, 0x20636f, 0x206465, 0x20656c, 0x20656e, 0x206573, 0x20696e, 0x206c61, 0x206c6f, 0x207061, 0x20706f, 0x207072, 0x207175, 0x207265, 0x207365, 0x20756e, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206c, 0x612070, 0x616369, 0x61646f, 0x616c20, 0x617220, 0x617320, 0x6369f3, 0x636f6e, 0x646520, 0x64656c, 0x646f20, 0x652064, 0x652065, 0x65206c, 0x656c20, 0x656e20, 0x656e74, 0x657320, 0x657374, 0x69656e, 0x69f36e, 0x6c6120, 0x6c6f73, 0x6e2065, 0x6e7465, 0x6f2064, 0x6f2065, 0x6f6e20, 0x6f7220, 0x6f7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746f20, 0x756520, 0xf36e20]), new NGramsPlusLang('fr', [0x206175, 0x20636f, 0x206461, 0x206465, 0x206475, 0x20656e, 0x206574, 0x206c61, 0x206c65, 0x207061, 0x20706f, 0x207072, 0x207175, 0x207365, 0x20736f, 0x20756e, 0x20e020, 0x616e74, 0x617469, 0x636520, 0x636f6e, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206c, 0x652070, 0x652073, 0x656e20, 0x656e74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696f6e, 0x697320, 0x697420, 0x6c6120, 0x6c6520, 0x6c6573, 0x6d656e, 0x6e2064, 0x6e6520, 0x6e7320, 0x6e7420, 0x6f6e20, 0x6f6e74, 0x6f7572, 0x717565, 0x72206c, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206c, 0x732070, 0x742064, 0x746520, 0x74696f, 0x756520, 0x757220]), new NGramsPlusLang('it', [0x20616c, 0x206368, 0x20636f, 0x206465, 0x206469, 0x206520, 0x20696c, 0x20696e, 0x206c61, 0x207065, 0x207072, 0x20756e, 0x612063, 0x612064, 0x612070, 0x612073, 0x61746f, 0x636865, 0x636f6e, 0x64656c, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206c, 0x652070, 0x652073, 0x656c20, 0x656c6c, 0x656e74, 0x657220, 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696c20, 0x696e20, 0x696f6e, 0x6c6120, 0x6c6520, 0x6c6920, 0x6c6c61, 0x6e6520, 0x6e6920, 0x6e6f20, 0x6e7465, 0x6f2061, 0x6f2064, 0x6f2069, 0x6f2073, 0x6f6e20, 0x6f6e65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746f20, 0x7a696f]), new NGramsPlusLang('nl', [0x20616c, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656e, 0x206765, 0x206865, 0x20696e, 0x206d61, 0x206d65, 0x206f70, 0x207465, 0x207661, 0x207665, 0x20766f, 0x207765, 0x207a69, 0x61616e, 0x616172, 0x616e20, 0x616e64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656e, 0x646572, 0x652062, 0x652076, 0x65656e, 0x656572, 0x656e20, 0x657220, 0x657273, 0x657420, 0x67656e, 0x686574, 0x696520, 0x696e20, 0x696e67, 0x697320, 0x6e2062, 0x6e2064, 0x6e2065, 0x6e2068, 0x6e206f, 0x6e2076, 0x6e6465, 0x6e6720, 0x6f6e64, 0x6f6f72, 0x6f7020, 0x6f7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656e, 0x746572, 0x76616e, 0x766572, 0x766f6f]), new NGramsPlusLang('no', [0x206174, 0x206176, 0x206465, 0x20656e, 0x206572, 0x20666f, 0x206861, 0x206920, 0x206d65, 0x206f67, 0x2070e5, 0x207365, 0x20736b, 0x20736f, 0x207374, 0x207469, 0x207669, 0x20e520, 0x616e64, 0x617220, 0x617420, 0x646520, 0x64656e, 0x646574, 0x652073, 0x656420, 0x656e20, 0x656e65, 0x657220, 0x657265, 0x657420, 0x657474, 0x666f72, 0x67656e, 0x696b6b, 0x696c20, 0x696e67, 0x6b6520, 0x6b6b65, 0x6c6520, 0x6c6c65, 0x6d6564, 0x6d656e, 0x6e2073, 0x6e6520, 0x6e6720, 0x6e6765, 0x6e6e65, 0x6f6720, 0x6f6d20, 0x6f7220, 0x70e520, 0x722073, 0x726520, 0x736f6d, 0x737465, 0x742073, 0x746520, 0x74656e, 0x746572, 0x74696c, 0x747420, 0x747465, 0x766572]), new NGramsPlusLang('pt', [0x206120, 0x20636f, 0x206461, 0x206465, 0x20646f, 0x206520, 0x206573, 0x206d61, 0x206e6f, 0x206f20, 0x207061, 0x20706f, 0x207072, 0x207175, 0x207265, 0x207365, 0x20756d, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646f, 0x616c20, 0x617220, 0x617261, 0x617320, 0x636f6d, 0x636f6e, 0x646120, 0x646520, 0x646f20, 0x646f73, 0x652061, 0x652064, 0x656d20, 0x656e74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6d656e, 0x6e7465, 0x6e746f, 0x6f2061, 0x6f2063, 0x6f2064, 0x6f2065, 0x6f2070, 0x6f7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746f20, 0x756520, 0xe36f20, 0xe7e36f]), new NGramsPlusLang('sv', [0x206174, 0x206176, 0x206465, 0x20656e, 0x2066f6, 0x206861, 0x206920, 0x20696e, 0x206b6f, 0x206d65, 0x206f63, 0x2070e5, 0x20736b, 0x20736f, 0x207374, 0x207469, 0x207661, 0x207669, 0x20e472, 0x616465, 0x616e20, 0x616e64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656e, 0x646572, 0x646574, 0x656420, 0x656e20, 0x657220, 0x657420, 0x66f672, 0x67656e, 0x696c6c, 0x696e67, 0x6b6120, 0x6c6c20, 0x6d6564, 0x6e2073, 0x6e6120, 0x6e6465, 0x6e6720, 0x6e6765, 0x6e696e, 0x6f6368, 0x6f6d20, 0x6f6e20, 0x70e520, 0x722061, 0x722073, 0x726120, 0x736b61, 0x736f6d, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696c, 0x747420, 0x766172, 0xe47220, 0xf67220])];
}
}, {
key: "name",
value: function name(det) {
return det && det.fC1Bytes ? 'windows-1252' : 'ISO-8859-1';
}
}]);
return ISO_8859_1;
}(sbcs);
var ISO_8859_2 =
/*#__PURE__*/
function (_sbcs2) {
_inherits(ISO_8859_2, _sbcs2);
function ISO_8859_2() {
_classCallCheck(this, ISO_8859_2);
return _possibleConstructorReturn(this, _getPrototypeOf(ISO_8859_2).apply(this, arguments));
}
_createClass(ISO_8859_2, [{
key: "byteMap",
value: function byteMap() {
return [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0x20, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0x20, 0xb1, 0x20, 0xb3, 0x20, 0xb5, 0xb6, 0xb7, 0x20, 0xb9, 0xba, 0xbb, 0xbc, 0x20, 0xbe, 0xbf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x20, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0x20];
}
}, {
key: "ngrams",
value: function ngrams() {
return [new NGramsPlusLang('cs', [0x206120, 0x206279, 0x20646f, 0x206a65, 0x206e61, 0x206e65, 0x206f20, 0x206f64, 0x20706f, 0x207072, 0x2070f8, 0x20726f, 0x207365, 0x20736f, 0x207374, 0x20746f, 0x207620, 0x207679, 0x207a61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656d20, 0x656eed, 0x686f20, 0x686f64, 0x697374, 0x6a6520, 0x6b7465, 0x6c6520, 0x6c6920, 0x6e6120, 0x6ee920, 0x6eec20, 0x6eed20, 0x6f2070, 0x6f646e, 0x6f6a69, 0x6f7374, 0x6f7520, 0x6f7661, 0x706f64, 0x706f6a, 0x70726f, 0x70f865, 0x736520, 0x736f75, 0x737461, 0x737469, 0x73746e, 0x746572, 0x746eed, 0x746f20, 0x752070, 0xbe6520, 0xe16eed, 0xe9686f, 0xed2070, 0xed2073, 0xed6d20, 0xf86564]), new NGramsPlusLang('hu', [0x206120, 0x20617a, 0x206265, 0x206567, 0x20656c, 0x206665, 0x206861, 0x20686f, 0x206973, 0x206b65, 0x206b69, 0x206bf6, 0x206c65, 0x206d61, 0x206d65, 0x206d69, 0x206e65, 0x20737a, 0x207465, 0x20e973, 0x612061, 0x61206b, 0x61206d, 0x612073, 0x616b20, 0x616e20, 0x617a20, 0x62616e, 0x62656e, 0x656779, 0x656b20, 0x656c20, 0x656c65, 0x656d20, 0x656e20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686f67, 0x696e74, 0x697320, 0x6b2061, 0x6bf67a, 0x6d6567, 0x6d696e, 0x6e2061, 0x6e616b, 0x6e656b, 0x6e656d, 0x6e7420, 0x6f6779, 0x732061, 0x737a65, 0x737a74, 0x737ae1, 0x73e967, 0x742061, 0x747420, 0x74e173, 0x7a6572, 0xe16e20, 0xe97320]), new NGramsPlusLang('pl', [0x20637a, 0x20646f, 0x206920, 0x206a65, 0x206b6f, 0x206d61, 0x206d69, 0x206e61, 0x206e69, 0x206f64, 0x20706f, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779, 0x207a20, 0x207a61, 0x612070, 0x612077, 0x616e69, 0x636820, 0x637a65, 0x637a79, 0x646f20, 0x647a69, 0x652070, 0x652073, 0x652077, 0x65207a, 0x65676f, 0x656a20, 0x656d20, 0x656e69, 0x676f20, 0x696120, 0x696520, 0x69656a, 0x6b6120, 0x6b6920, 0x6b6965, 0x6d6965, 0x6e6120, 0x6e6961, 0x6e6965, 0x6f2070, 0x6f7761, 0x6f7769, 0x706f6c, 0x707261, 0x70726f, 0x70727a, 0x727a65, 0x727a79, 0x7369ea, 0x736b69, 0x737461, 0x776965, 0x796368, 0x796d20, 0x7a6520, 0x7a6965, 0x7a7920, 0xf37720]), new NGramsPlusLang('ro', [0x206120, 0x206163, 0x206361, 0x206365, 0x20636f, 0x206375, 0x206465, 0x206469, 0x206c61, 0x206d61, 0x207065, 0x207072, 0x207365, 0x2073e3, 0x20756e, 0x20ba69, 0x20ee6e, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636f6e, 0x637520, 0x63e320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070, 0x652073, 0x656120, 0x656920, 0x656c65, 0x656e74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696e20, 0x6c6120, 0x6c6520, 0x6c6f72, 0x6c7569, 0x6e6520, 0x6e7472, 0x6f7220, 0x70656e, 0x726520, 0x726561, 0x727520, 0x73e320, 0x746520, 0x747275, 0x74e320, 0x756920, 0x756c20, 0xba6920, 0xee6e20])];
}
}, {
key: "name",
value: function name(det) {
return det && det.fC1Bytes ? 'windows-1250' : 'ISO-8859-2';
}
}]);
return ISO_8859_2;
}(sbcs);
var ISO_8859_5 =
/*#__PURE__*/
function (_sbcs3) {
_inherits(ISO_8859_5, _sbcs3);
function ISO_8859_5() {
_classCallCheck(this, ISO_8859_5);
return _possibleConstructorReturn(this, _getPrototypeOf(ISO_8859_5).apply(this, arguments));
}
_createClass(ISO_8859_5, [{
key: "byteMap",
value: function byteMap() {
return [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0x20, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x20, 0xfe, 0xff];
}
}, {
key: "ngrams",
value: function ngrams() {
return [0x20d220, 0x20d2de, 0x20d4de, 0x20d7d0, 0x20d820, 0x20dad0, 0x20dade, 0x20ddd0, 0x20ddd5, 0x20ded1, 0x20dfde, 0x20dfe0, 0x20e0d0, 0x20e1de, 0x20e1e2, 0x20e2de, 0x20e7e2, 0x20ede2, 0xd0ddd8, 0xd0e2ec, 0xd3de20, 0xd5dbec, 0xd5ddd8, 0xd5e1e2, 0xd5e220, 0xd820df, 0xd8d520, 0xd8d820, 0xd8ef20, 0xdbd5dd, 0xdbd820, 0xdbecdd, 0xddd020, 0xddd520, 0xddd8d5, 0xddd8ef, 0xddde20, 0xddded2, 0xde20d2, 0xde20df, 0xde20e1, 0xded220, 0xded2d0, 0xded3de, 0xded920, 0xdedbec, 0xdedc20, 0xdee1e2, 0xdfdedb, 0xdfe0d5, 0xdfe0d8, 0xdfe0de, 0xe0d0d2, 0xe0d5d4, 0xe1e2d0, 0xe1e2d2, 0xe1e2d8, 0xe1ef20, 0xe2d5db, 0xe2de20, 0xe2dee0, 0xe2ec20, 0xe7e2de, 0xebe520];
}
}, {
key: "name",
value: function name() {
return 'ISO-8859-5';
}
}, {
key: "language",
value: function language() {
return 'ru';
}
}]);
return ISO_8859_5;
}(sbcs);
var ISO_8859_6 =
/*#__PURE__*/
function (_sbcs4) {
_inherits(ISO_8859_6, _sbcs4);
function ISO_8859_6() {
_classCallCheck(this, ISO_8859_6);
return _possibleConstructorReturn(this, _getPrototypeOf(ISO_8859_6).apply(this, arguments));
}
_createClass(ISO_8859_6, [{
key: "byteMap",
value: function byteMap() {
return [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,