tafic
Version:
To ASCII First Impression Converter - normalizes text to closest first impression of reader based on ASCII set.
287 lines (283 loc) • 18 kB
JavaScript
;
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var index_exports = {};
__export(index_exports, {
TAFIC: () => TAFIC,
default: () => TAFIC
});
module.exports = __toCommonJS(index_exports);
var import_grapheme = require("unicode-segmenter/grapheme");
var import_homoglypher = __toESM(require("homoglypher"), 1);
var import_emoji_regex = __toESM(require("emoji-regex"), 1);
// src/tools/HardCodedMap/index.ts
var HardCoded = [
{ "replacement": "_", "similarities": ["\u02F7", "\u02FD", "\u2582"] },
{ "replacement": "-", "similarities": ["\uA7F7", "\u301C", "\xAC", "\u058A", "\u1169", "\u1173", "\u2014", "\u2322", "\u2500", "\u2501", "\u257E", "\u25AA", "\u30FC", "\u2E3A"] },
{ "replacement": ";", "similarities": ["\u204F"] },
{ "replacement": ":", "similarities": ["\u2803", "\u280C", "\u2830", "\u2848", "\u205D"] },
{ "replacement": "!", "similarities": ["\u2757", "\u2757\uFE0F", "\u2763", "\u28DB"] },
{ "replacement": "?", "similarities": ["\xBF", "\u061F"] },
{ "replacement": ".", "similarities": ["\u2B1D", "\u2219", "\u2981", "\u16EB", "\u2027", "\u2E33", "\uA78F", "\xB7", "\u02D1", "\u02EF", "\u02F3", "\u2022", "\u22C5", "\u2804", "\u2810", "\u2820", "\u2840", "\u2880", "\u2E31", "\u3002", "\u30FB", "\uFF61"] },
{ "replacement": ",", "similarities": ["\uFF64", "\u0709", "\u2E34", "\u071D", "\xB8", "\u02CF", "\u02D2", "\u02D3", "\u02D4", "\u02EC", "\u02F0", "\u02F1", "\u02F2", "\u02FB", "\u02FC", "\u02FF", "\u2884", "\u28A0", "\u28A4", "\u28C4", "\u28E0", "\u3001"] },
{ "replacement": "'", "similarities": ["\u17F8", "\u02D9", "\u02F9", "\u02FA", "\u2801", "\u2808", "\u280A", "\u280B", "\u2818", "\u2819"] },
{ "replacement": '"', "similarities": ["\u02F5", "\u02F6", "\u201D", "\u201E"] },
{ "replacement": "(", "similarities": ["\u3010"] },
{ "replacement": ")", "similarities": ["\u3011"] },
{ "replacement": "[", "similarities": ["\u298F"] },
{ "replacement": "]", "similarities": ["\u298E"] },
{ "replacement": "*", "similarities": ["\u0489", "\u203B", "\u2600", "\u2606", "\u26E5", "\u2729", "\u272E", "\u2731"] },
{ "replacement": "/", "similarities": ["\u0F3C"] },
{ "replacement": "\\", "similarities": ["\u0F3D"] },
{ "replacement": "%", "similarities": ["\xF7"] },
{ "replacement": "^", "similarities": ["\u1109", "\u110C"] },
{ "replacement": "<", "similarities": ["\u15D2", "\u3008", "\u300A"] },
{ "replacement": "=", "similarities": ["\u02ED", "\u2550", "\u30CB"] },
{ "replacement": ">", "similarities": ["\u15D5", "\u3009", "\u300B"] },
{ "replacement": "|", "similarities": ["\u09F7", "\u23D0", "\xA6", "\u1161", "\u1163", "\u1165", "\u1175", "\u254E", "\u2807", "\u2838", "\u2846", "\u2847", "\u28B0", "\u28B8", "\u22EE", "\uFE34"] },
{ "replacement": "$", "similarities": ["\u{1F4B2}", "\u0586"] },
{ "replacement": "0", "similarities": ["0\uFE0F\u20E3", "0\u20E3", "\u24FF", "\u{1F10C}", "\u24EA", "\u{1F10B}", "\u03B8", "\u11B7", "\u1C50", "\u56D7"] },
{ "replacement": "1", "similarities": ["1\uFE0F\u20E3", "1\u20E3", "\u2776", "\u278A", "\u2460", "\u2780"] },
{ "replacement": "2", "similarities": ["2\uFE0F\u20E3", "2\u20E3", "\u2777", "\u278B", "\u2461", "\u2781", "\u01A8", "\u1105", "\u11AF", "\u5DF1"] },
{ "replacement": "3", "similarities": ["3\uFE0F\u20E3", "3\u20E3", "\u2778", "\u278C", "\u2462", "\u2782", "\u042D", "\u0437", "\u044D", "\u2128", "\uA76B"] },
{ "replacement": "4", "similarities": ["4\uFE0F\u20E3", "4\u20E3", "\u2779", "\u278D", "\u2463", "\u2783"] },
{ "replacement": "5", "similarities": ["5\uFE0F\u20E3", "5\u20E3", "\u277A", "\u278E", "\u2464", "\u2784", "\u01BD", "\u28EB", "\u4E02"] },
{ "replacement": "6", "similarities": ["6\uFE0F\u20E3", "6\u20E3", "\u277B", "\u278F", "\u2465", "\u2785"] },
{ "replacement": "7", "similarities": ["7\uFE0F\u20E3", "7\u20E3", "\u277C", "\u2790", "\u2466", "\u2786", "\u30AF", "\u30BF", "\u30EF"] },
{ "replacement": "8", "similarities": ["8\uFE0F\u20E3", "8\u20E3", "\u277D", "\u2791", "\u2467", "\u2787", "\u2F47"] },
{ "replacement": "9", "similarities": ["9\uFE0F\u20E3", "9\u20E3", "\u277E", "\u2792", "\u2468", "\u2788", "\u28FB"] },
{ "replacement": "a", "similarities": ["\u1D00", "\u1D8F", "\u2376", "\u25B4", "\uAB30", "\u25B2"] },
{ "replacement": "A", "similarities": ["\u2A5C", "\u15CB", "\u{1F1E6}", "\u0394", "\u039B", "\u03BB", "\u0434", "\u0E04", "\u13AF", "\u15E9", "\u20B3", "\u4EBA", "\u{10300}", "\u{1F170}", "\u{1F170}\uFE0F"] },
{ "replacement": "b", "similarities": ["\u0180", "\u0183", "\u0253", "\u0299", "\u042A", "\u0432", "\u044A", "\u044C", "\u1D03", "\u1D6C", "\u1D80", "\u2C83", "\u3061"] },
{ "replacement": "B", "similarities": ["\xDF", "\u0181", "\u0243", "\u0E3F", "\u136A", "\u2C82", "\u4E43", "\u{1F171}\uFE0F"] },
{ "replacement": "c", "similarities": ["\xA2", "\xA9", "\u0188", "\u0255", "\u03C2", "\u03F2", "\u1103", "\uFFE0"] },
{ "replacement": "C", "similarities": ["\u0187", "\u0297", "\u03F9", "\u122D", "\u13E3", "\u1455", "\u20B5", "\u2282", "\u531A", "\uA792", "\u{1F172}"] },
{ "replacement": "d", "similarities": ["\u0111", "\u0221", "\u0256", "\u0257", "\u0500", "\u1D05", "\u1D6D", "\u1D81", "\u1D91"] },
{ "replacement": "D", "similarities": ["\u2AD0", "\u{1F1E9}", "\xD0", "\u0110", "\u018A"] },
{ "replacement": "e", "similarities": ["\u0250", "\u2D12", "\u01DD", "\u0259", "\u1D92", "\u2C78"] },
{ "replacement": "E", "similarities": ["\u2208", "\u{1F1EA}", "\xA3", "\u018E", "\u0190", "\u025B", "\u039E", "\u03AD", "\u03B5", "\u03F5", "\u0404", "\u0454", "\u0639", "\u15F4", "\u1653", "\u1D07", "\u1D93", "\u20AC", "\u220D", "\u224B", "\u2C89", "\u4E47", "\u{10304}"] },
{ "replacement": "f", "similarities": ["\u0192", "\u0493", "\u1D82", "\uA730"] },
{ "replacement": "F", "similarities": ["\u0191", "\uA77B"] },
{ "replacement": "g", "similarities": ["\u13FD", "\u01E5", "\u0260", "\u0262"] },
{ "replacement": "G", "similarities": ["\u13F5", "\u{1F1EC}", "\u0193", "\u01E4", "\u20B2"] },
{ "replacement": "h", "similarities": ["\u0127", "\u0266", "\u029C", "\u2C68", "\u3093"] },
{ "replacement": "H", "similarities": ["\u04CA", "\u043D", "\u1162", "\u2C67"] },
{ "replacement": "i", "similarities": ["\xA1", "\u0268", "\u1D96", "\u2AEF", "\u2C93", "\uAAB1"] },
{ "replacement": "I", "similarities": ["\u0197", "\u0406", "\u0407", "\u5DE5"] },
{ "replacement": "j", "similarities": ["\u0237", "\u0249", "\u025F", "\u029D", "\u062F", "\u0631", "\u1D0A", "\u2878", "\u28E8"] },
{ "replacement": "J", "similarities": ["\u1499", "\u0248", "\u0643", "\u14DA", "\u256F", "\u28FC", "\u4E01"] },
{ "replacement": "k", "similarities": ["\u0199", "\u03BA", "\u043A", "\u045C", "\u049D", "\u049F", "\u04C4", "\u1D0B", "\u1D84", "\u2C6A", "\u2C95", "\uA741", "\uA743", "\uA745"] },
{ "replacement": "K", "similarities": ["\u0198", "\u049C", "\u20AD", "\u2C69", "\uA740", "\uA742"] },
{ "replacement": "l", "similarities": ["\u1DA9", "\u026D", "\u{1044A}", "\u0140", "\u0142", "\u019A", "\u0234", "\u026B", "\u026C", "\u026D", "\u029F", "\u1963", "\u1D0C", "\u1D85", "\u2570", "\u2759", "\u275A", "\u2C61", "\uA749", "\uA781"] },
{ "replacement": "L", "similarities": ["\u056C", "\u2517", "\u0B9F", "\u14B7", "\u{10443}", "\u013F", "\u0141", "\u023D", "\u053C", "\u1102", "\u11AB", "\u221F", "\u2514", "\u2827", "\u283C", "\u28C6", "\u28C7", "\u28E7", "\u28F8", "\u2C60", "\u2C62", "\u3057", "\u3125", "\uA746", "\uA748", "\u{1030B}", "\u{1F1F1}", "\u14BA"] },
{ "replacement": "m", "similarities": ["\u2AD9", "\u04CE", "\u0271", "\u043C", "\u0E53", "\u1D0D", "\u1D6F", "\u1D86", "\u20A5"] },
{ "replacement": "M", "similarities": ["\u{102BF}", "\u03FB", "\u04CD", "\u164F", "\u028D", "\u16D7", "\u2C6E", "\uAB87", "\u{1028E}", "\u{1033C}", "\u{1F1F2}"] },
{ "replacement": "n", "similarities": ["\u0149", "\u014B", "\u019E", "\u0235", "\u0272", "\u0273", "\u0274", "\u03AE", "\u03B7", "\u03C0", "\u0438", "\u043B", "\u043F", "\u0572", "\u0E20", "\u1D70", "\u1D87", "\u1F94", "\u2C9B"] },
{ "replacement": "N", "similarities": ["\u019D", "\u0220", "\u041F", "\u0E96", "\u1609", "\u20A6", "\uFB21"] },
{ "replacement": "o", "similarities": ["\u{1F1F4}", "\u0298", "\xB0", "\xF0", "\xF8", "\u01FF", "\u0275", "\u0301", "\u03B4", "\u0488", "\u0FC1", "\u1106", "\u110B", "\u2218", "\u25E6", "\u25AB", "\u25CE", "\u25CF", "\u25D5", "\u2688", "\u274D", "\u2C7A", "\u30ED", "\uA74B", "\uA74D"] },
{ "replacement": "O", "similarities": ["\xD8", "\u019F", "\u01FE", "\u2B58", "\u25EF", "\uA74A", "\uA74C", "\u{11136}"] },
{ "replacement": "p", "similarities": ["\u2117", "\xFE", "\u01A5", "\u01BF", "\u048F", "\u0584", "\u1D18", "\u1D71", "\u1D7D", "\u1D88", "\u281F", "\uA751", "\uA753", "\uA755"] },
{ "replacement": "P", "similarities": ["\u10B2", "\u1477", "\u1598", "\xDE", "\u01A4", "\u01F7", "\u048E", "\u05E7", "\u0645", "\u146C", "\u1575", "\u15B0", "\u203D", "\u20B1", "\u2118", "\u285F", "\u287F", "\u2C63", "\u5369", "\u5C38", "\uA750", "\uA752", "\uA754", "\u{1F17F}\uFE0F", "\u{1F1F5}"] },
{ "replacement": "q", "similarities": ["\xB6", "\u024B", "\u1D90", "\uA757", "\uA7FC"] },
{ "replacement": "r", "similarities": ["\xAE", "\u017F", "\u024D", "\u027C", "\u027D", "\u027E", "\u0280", "\u0413", "\u0453", "\u0491", "\u136B", "\u1D72", "\u1D73", "\u1D89", "\u1E9B", "\u280F", "\u284F", "\uA785"] },
{ "replacement": "R", "similarities": ["\u024C", "\u042F", "\u044F", "\u2C64", "\u5C3A", "\u{1F1F7}"] },
{ "replacement": "s", "similarities": ["\u023F", "\u0282", "\u064A", "\u1D74", "\u1D8A", "\u2E09"] },
{ "replacement": "S", "similarities": ["\xA7", "\u0C15", "\u0D1F", "\u10BD", "\u27C6", "\u{10314}", "\u{1F182}"] },
{ "replacement": "t", "similarities": ["\u0167", "\u01AB", "\u01AD", "\u0236", "\u0288", "\u03C4", "\u0442", "\u0567", "\u15B6", "\u1D1B", "\u1D75", "\u2020", "\u271D", "\u2CA7", "\u30C6"] },
{ "replacement": "T", "similarities": ["\u252C", "\u23C9", "\u1350", "\u{1F1F9}", "\u0166", "\u01AC", "\u01AE", "\u0372", "\u0373", "\u1370", "\u20AE", "\u2564", "\u2566", "\u2602", "\u3012", "\u3059"] },
{ "replacement": "u", "similarities": ["\u028A", "\u0446", "\u0574", "\u062A", "\u0646", "\u0E22", "\u1959", "\u1D17", "\u1D7E"] },
{ "replacement": "U", "similarities": ["\u0BAA", "\uA4A4", "\u0244", "\u0531", "\u0E19", "\u1371", "\u1458", "\u160E", "\u22D3", "\u{1F1FA}"] },
{ "replacement": "v", "similarities": ["\u2C71", "\u2C74"] },
{ "replacement": "V", "similarities": ["\u2207", "\u2713", "\u2714", "\u2714\uFE0F"] },
{ "replacement": "w", "similarities": ["\u03C9", "\u2C73"] },
{ "replacement": "W", "similarities": ["\u0429", "\u0448", "\u0449", "\u20A9", "\u2C72", "\uA7FD"] },
{ "replacement": "x", "similarities": ["\u0436", "\u10EF", "\u1D8D"] },
{ "replacement": "X", "similarities": ["\u04FC", "\u4E42"] },
{ "replacement": "y", "similarities": ["\u2442", "\u2D24", "\u01B4", "\u024F"] },
{ "replacement": "Y", "similarities": ["\u07CC", "\xA5", "\u01B3", "\u024E", "\u040E", "\u1EFE"] },
{ "replacement": "z", "similarities": ["\u{1F1FF}", "\u01B6", "\u0225", "\u0240", "\u0290", "\u0291", "\u1D76", "\u1D8E", "\u2C6C", "\u2C8D"] },
{ "replacement": "Z", "similarities": ["\u01B5", "\u0224", "\u10CD", "\u136F", "\u2621", "\u2C6B", "\u2C7F", "\u2C8C", "\u4E59"] },
{ "replacement": "->", "similarities": ["\u2192", "\u27F6"] },
{ "replacement": "=>", "similarities": ["\u27F9"] },
{ "replacement": ", ", "similarities": ["\u3001"] },
{ "replacement": "::", "similarities": ["\u28D2"] },
{ "replacement": ":.", "similarities": ["\u28C1"] },
{ "replacement": "..", "similarities": ["\u2824", "\u28C0"] },
{ "replacement": "''", "similarities": ["\u2809"] },
{ "replacement": "/\\", "similarities": ["\u30CF"] },
{ "replacement": "<-", "similarities": ["\u2190"] },
{ "replacement": "=)", "similarities": ["\u30B7", "\u30C3"] },
{ "replacement": "---", "similarities": ["\u2E3B"] },
{ "replacement": "AB", "similarities": ["\u{1F18E}"] },
{ "replacement": "bl", "similarities": ["\u044B"] },
{ "replacement": "cl", "similarities": ["\uB514"] },
{ "replacement": "CL", "similarities": ["\u{1F191}"] },
{ "replacement": "COOL", "similarities": ["\u{1F192}"] },
{ "replacement": "FREE", "similarities": ["\u{1F193}"] },
{ "replacement": "ID", "similarities": ["\u{1F194}"] },
{ "replacement": "ij", "similarities": ["\u2E89", "\u30EA"] },
{ "replacement": "io", "similarities": ["\u044E"] },
{ "replacement": "lt", "similarities": ["\u3051"] },
{ "replacement": "NEW", "similarities": ["\u{1F195}"] },
{ "replacement": "NG", "similarities": ["\u{1F196}"] },
{ "replacement": "OK", "similarities": ["\u{1F197}"] },
{ "replacement": "RU", "similarities": ["\u{1F1F7}\u{1F1FA}"] },
{ "replacement": "SOS", "similarities": ["\u{1F198}"] },
{ "replacement": "UP!", "similarities": ["\u{1F199}"] },
{ "replacement": "VS", "similarities": ["\u{1F19A}"] }
];
var HardCodedMap = (() => {
const map = /* @__PURE__ */ new Map();
for (const unit of HardCoded) {
for (const similiarity of unit.similarities)
map.set(similiarity, unit.replacement);
}
return map;
})();
var HardCodedMap_default = HardCodedMap;
// src/tools/ZeroWidthRegExp.ts
var Data = [
["200B", "200D"],
// ZW chars
"200E",
["1BCA0", "1BCA3"],
// Shorthand controls
["FFF9", "FFFB"],
// Interlinear annotation
"034F",
"061C",
"115F",
"1160",
"17B4",
"17B5",
"180E",
"200C",
"2060",
"2061",
"2062",
"2063",
"2028",
"2029",
"202F",
"2800",
"3164",
"FEFF",
"FFA0",
"FE0F"
// Variation selector - forces emoji representation e.g. "❗️" becomes two chars: emoji + FE0F
];
var ZeroWidthRegExp_default = (() => {
const pattern = Data.map((entry) => {
if (Array.isArray(entry)) {
const [start, end] = entry;
return `\\u{${start}}-\\u{${end}}`;
} else {
return `\\u{${entry}}`;
}
}).join("");
return new RegExp(`[${pattern}]`, "gu");
})();
// src/index.ts
var DefaultOptions = {
skipHardcodedMapping: false,
removeLeftovers: true,
onLeftovers: null
};
var TAFIC = class {
static Homoglypher = new import_homoglypher.default();
static EmojiRegex = (0, import_emoji_regex.default)();
static RegExp_ALL_ASCII_IN_STR = /[\x00-\x7F]+/g;
static RegExp_ALL_NONASCII_IN_STR = /[^\x00-\x7F]+/g;
static RegExp_ACCENTS = /\p{Mn}/gu;
//
static Normalize(str, o) {
const skipHardcodedMapping = o?.skipHardcodedMapping ?? DefaultOptions.skipHardcodedMapping;
const removeLeftovers = o?.removeLeftovers ?? DefaultOptions.removeLeftovers;
str = this.SeparateAndRemoveDiacriticsForHCMapping(str);
let graphemes = this.GetGraphemes(str);
if (!skipHardcodedMapping)
graphemes = this.ApplyHardcodedMapRules(graphemes);
graphemes = this.RemoveZeroWidthCharacters(graphemes);
str = graphemes.join("");
str = this.ApplyBuiltinCompatibilities(str);
str = this.Homoglypher.normalize(str, { skipCustom: true });
if (o?.onLeftovers) {
this.RegExp_ALL_ASCII_IN_STR.lastIndex = 0;
const nonASCII = str.replace(this.RegExp_ALL_ASCII_IN_STR, "");
if (nonASCII.length > 0)
o.onLeftovers(nonASCII);
}
const result = (removeLeftovers ? this.RemoveNonASCII(str) : str).trim();
return result;
}
//
static GetGraphemes(str) {
const segments = (0, import_grapheme.graphemeSegments)(str);
const result = [];
for (const s of segments)
result.push(s.segment);
return result;
}
static IsGraphemeOnlyASCII(grapheme) {
this.RegExp_ALL_NONASCII_IN_STR.lastIndex = 0;
return !this.RegExp_ALL_NONASCII_IN_STR.test(grapheme);
}
static IsStringOnlyEmoji(str) {
const matches = str.match(this.EmojiRegex);
return matches !== null && matches.join("") === str;
}
//
static SeparateAndRemoveDiacriticsForHCMapping(str) {
str = str.normalize("NFD");
this.RegExp_ACCENTS.lastIndex = 0;
return str.replace(this.RegExp_ACCENTS, "");
}
static ApplyHardcodedMapRules(graphemes) {
const hcmapMappedGlyphs = graphemes.map((g) => {
return HardCodedMap_default.get(g) ?? g;
});
return hcmapMappedGlyphs;
}
static RemoveZeroWidthCharacters(graphemes) {
return graphemes.map((g) => {
if (this.IsGraphemeOnlyASCII(g) || this.IsStringOnlyEmoji(g))
return g;
return g.replace(ZeroWidthRegExp_default, "");
});
}
static ApplyBuiltinCompatibilities(str) {
return str.normalize("NFKC");
}
static RemoveNonASCII(str) {
this.RegExp_ALL_NONASCII_IN_STR.lastIndex = 0;
return str.replace(this.RegExp_ALL_NONASCII_IN_STR, "");
}
};
module.exports = module.exports.default;
//# sourceMappingURL=index.cjs.map