budoux
Version:
A small chunk segmenter.
683 lines (677 loc) • 69.1 kB
JavaScript
"use strict";
(() => {
// module/data/models/ja.js
var model = { "UW3": { "\u3002": 6699, "\u306B": 3897, "\u3001": 4784, "\u306E": 3706, "\u304C": 4162, "\u3092": 5769, "\u306F": 4221, "\u3068": 2900, "\u308B": 2971, "\u304F": 2902, "\u3082": 3155, "\u3046": 2242, "\u3066": 2447, "\u3067": 3372, "\u305F": 2217, "\u3044": 1855, "\u306A": 2357, "\u3063": -1853, "\u65B9": 2814, "\u3089": 1353, "\u3093": 967, "\u3070": 1790, "\uFF3D": 2166, "\u304B": 1899, "\uFF0C": 3358, "\u308A": 1013, "\uFF01": 3252, "\u30FC": -102, "\u3084": 2149, "\uFF1F": 3703, "\uFF0E": 2093, "\u30FB": 1074, "\u2026": 2428, "\u90E8": 1732, "\u3069": 992, "\u3054": -2327, "\uFF09": 2573, "\u30C3": -2137, "\u2606": 2675, "\u304A": -1495, "\u4E0A": 428, "\u4EE5": -1870, "\u3064": 1078, "\u300D": 1608, "\u266A": 2714, "\u96FB": -1669, "\u30AF": 335, "\u3060": 866, "\u3078": 2073, "\u308D": 1142, "\u7B11": 2335, "\u305A": 1456, "\u9762": 475, "\u3073": 514, "\u3042": 1179, "\u4ECA": 1080, "\u3050": 2053, "\u5206": 1111, "\u611F": 1389, "\u5927": -1134, "\u4EAC": -1396, "\u4E2D": 998, "\u4EBA": 486, "\u5834": -794, "\u30A4": -568, "\u3059": 938, "\u3055": -599, "\u307E": -370, "\u5229": -681, "\u9593": 752, "\uFF10": -870, "\u3041": 1322, "\u30D5": -1285, "\u30F3": -284, "\u300C": -1434, "\u5F53": -337, "\u304D": 530, "\u65E5": 713, "\u30B9": -901, "\u3083": 1112, "\u5F8C": 1224, "\u3005": 1435, "\u30B7": -1196, "\u5EA6": 982, "\u89B3": -976, "\u98DF": -517, "\u884C": -725, "\u30C9": -995, "\u30EA": -636, "\u3048": 728, "\u7A7A": 902, "\u591C": 1480, "\u30A2": -700, "\u300F": 1404, "\uFF11": -550, "\u4F53": 1e3, "\u79CB": 1464, "\u5165": -819, "\u30D7": -718, "\u6614": 1287, "\uFF3B": -960, "\u984C": 1113, "\u80FD": -710, "\u7686": 1294, "\u3080": 920, "\u5B66": -552, "\u614B": -749, "\u6708": 584, "\u4FC2": 701, "\u5909": 328, "\u8D85": 976, "\u3076": 672, "\u7136": 515, "\u301C": 489, "\u308C": 295, "\u7531": 680, "\u6BB5": 13, "\u975E": 768, "\u5149": -218, "\u610F": -477, "\u305E": 825, "\u5E74": 344, "\u3079": -711, "\u6A5F": -463, "\u8A00": -419, "\u3061": 363, "\u6C17": 130, "\u307F": 275, "\uFF4F": -520, "\uFF1E": 588, "\u3057": 181, "\u3051": 159, "\u79FB": -635, "\u898B": 160, "\u30BA": 304, "\u81EA": -352, "\u679C": 767, "\u756A": 252, "\u8272": 469, "\u3000": 257, "\u3071": 454, "\u51FA": -160, "\u5C4B": 338, "\u30AB": -102, "\u30E1": -136, "\u6307": -240, "\u5E97": 203, "\u4E00": -50, "\u5225": 285, "\u982D": 486, "\u671D": 270, "\u60F3": 398, "\u5473": 101, "\u751F": -160, "\u8A9E": 159, "\u201D": 215, "\u7740": -149, "\u6240": 189, "\uFF1A": 94, "\u795E": -111, "\u5FDC": 149, "\u7A0B": 145, "\u4E0D": -93, "\u6599": -76, "\u5BFA": 50, "\u6C34": 54, "\u30EC": -54, "\u30D1": -29, "\u30B5": 33, "\u30B0": 12, "\u6709": -16 }, "UW4": { "\u3002": -7440, "\u306E": -4018, "\u304C": -3577, "\u306B": -4299, "\u3066": -3839, "\u308B": -5462, "\u3001": -7452, "\u306F": -3332, "\u3063": -4313, "\u3067": -3120, "\u3092": -4861, "\u3046": -2446, "\u3068": -2917, "\u308A": -4081, "\u307E": -1296, "\u305F": -1675, "\u3042": 501, "\u308C": -4326, "\u306A": -1928, "\u305D": 456, "\u304F": -2884, "\u304B": -2662, "\u3089": -4391, "\u3054": 591, "\u3057": -1725, "\u3053": -517, "\u3060": -2037, "\u304D": -2074, "\u3044": -1531, "\u3093": -3821, "\u30FC": -2899, "\u3082": -2746, "\u307F": -1111, "\u98DF": 652, "\u884C": 411, "\u601D": 1386, "\u3059": -1159, "\u7A7A": 2526, "\u5927": 788, "\u300D": -5393, "\u3058": -2352, "\u3084": -1257, "\u3051": -3441, "\u3081": -1867, "\u3055": -1716, "\u3088": -643, "\u300C": 2301, "\u8868": 1003, "\u65E5": 695, "\u5E83": 1811, "\u5834": 333, "\u4E00": 576, "\u6642": 564, "\u3061": -1762, "\u51B7": 2351, "\u4EAC": 614, "\uFF0C": -4205, "\u308F": -1624, "\u4EBA": 512, "\u79C1": 1957, "\uFF01": -4469, "\u3048": -1844, "\u30C3": -2189, "\u5F53": 384, "\u30FB": -677, "\u30F3": -2078, "\uFF11": 710, "\u65B9": -816, "\u3079": -2964, "\u624B": 270, "\u6765": 430, "\u308D": -2793, "\u305B": -1498, "\u30EB": -1178, "\u3070": -2432, "\u3064": -1195, "\u4F7F": 1131, "\uFF08": 2399, "\u6301": 180, "\u2026": -2208, "\u3086": -2265, "\uFF1F": -3412, "\uFF09": -4207, "\u3076": -2267, "\u4F55": 1291, "\u5F7C": 1247, "\u3073": -1619, "\u7B11": -3444, "\u672C": -1249, "\u3069": -697, "\uFF3B": 1760, "\uFF0E": -2874, "\u8A00": 227, "\u304A": 820, "\u305A": -1966, "\u30A4": -507, "\u4ECA": 1002, "\u3083": -2441, "\u898B": 386, "\u81EA": 792, "\uFF12": 533, "\u6700": 1072, "\u7528": -716, "\u5E2F": -1138, "\u9593": -393, "\u643A": 441, "\u591A": 1062, "\u30EA": -707, "\u80FD": -1270, "\u300F": -3209, "\uFF3D": -1427, "\u6C17": 235, "\u5165": 412, "\u6709": 744, "\u306D": -1659, "\u767D": 451, "\u3041": -1501, "\u304E": -1428, "\u4E2D": 34, "\u3072": 1076, "\u904A": 1409, "\u201D": -817, "\u98A8": 673, "\u30B3": 344, "\u3087": -1527, "\u6C34": 286, "\u60C5": -375, "\uFF10": -374, "\u3080": -1261, "\u5408": -948, "\u516C": 724, "\u307C": -949, "\u266A": -1513, "\u5C0F": 736, "\u50D5": 856, "\u2606": -1296, "\u301C": -1156, "\u30C9": -76, "\u697D": 308, "\u5168": 475, "\u66F8": 411, "\u30B9": -218, "\u4E8C": 481, "\u614B": -582, "\u5C11": 436, "\u4E0D": 505, "\u8FD4": -347, "\u5B9A": -356, "\u305C": -907, "\u9AD8": 229, "\u7279": 623, "\u796D": 413, "\u4ED6": 368, "\u3065": -776, "\u76EE": 117, "\u305E": -935, "\u8A71": 316, "\u5B50": -233, "\u3050": -840, "\u5E74": -625, "\uFF13": 22, "\u9053": 181, "\u7684": -500, "\uFF4F": -489, "\u5B9F": 368, "\u8CB7": 366, "\u7121": 192, "\u8857": 443, "\u6771": 470, "\u7269": -204, "\u30E5": -434, "\u307B": -176, "\u8FBC": -568, "\u7518": 383, "\u30BF": -212, "\u63DB": -278, "\u73FE": 191, "\u65B0": 150, "\u590F": 403, "\u30D5": -157, "\u30C8": -148, "\u3052": -455, "\u3056": -327, "\u30C7": 174, "\u5238": -471, "\u611F": 28, "\u79CB": 307, "\u96E3": 179, "\u5916": 153, "\u3078": -247, "\u30E7": -156, "\u6D41": 254, "\u821E": 138, "\u30B5": 75, "\u524D": -75, "\u5E97": 101, "\u62BC": 264, "\u958B": 61, "\u3000": -148, "\uFF3E": -256, "\u5EA6": -125, "\uFF15": 52, "\u3005": -142, "\u5199": 159, "\u56DE": -154, "\u30E9": -56, "\u95A2": -145, "\u5B89": 63, "\u76F8": 54, "\u53C2": 45, "\u767E": 45, "\u53D6": -45, "\u30E0": -41, "\u5C4B": -8, "\u901A": -8, "\u52DD": 4, "\u7F8E": 4 }, "UW5": { "\u3002": -2177, "\u3067": -955, "\u3059": -681, "\u3001": -1011, "\u306A": -608, "\u307E": -353, "\u3066": 169, "\u3063": 964, "\u308B": 175, "\u306F": -874, "\u304D": 1177, "\u3057": -394, "\u304A": 345, "\u3092": -674, "\u304F": 287, "\u3053": 500, "\u3058": 886, "\u306B": -612, "\u305D": -421, "\u80FD": 219, "\u308F": 710, "\u3079": 1871, "\u3048": 637, "\u3093": 1001, "\u3042": -271, "\u305F": 198, "\u307F": 293, "\u3044": 345, "\u30FB": -1082, "\u3068": -617, "\u3088": -587, "\u306E": -641, "\u308D": -360, "\u4E0A": -1087, "\u304B": 210, "\u3060": -338, "\u3081": 851, "\u5E2F": 303, "\u308C": 495, "\u3056": 1494, "\u3089": 222, "\u305A": 1173, "\u3055": -327, "\u3064": 1091, "\u30EB": -472, "\u9762": 776, "\u300D": -1068, "\u96FB": -285, "\u307C": 1415, "\u65E5": 342, "\u304C": -272, "\u3073": 826, "\u305B": -303, "\u601D": -891, "\u3061": 572, "\u51B7": 1123, "\u308A": 409, "\u5E74": 876, "\u4F7F": -1390, "\u5229": 365, "\u8449": 791, "\uFF01": -384, "\uFF3D": -470, "\u5408": 255, "\u3076": 15, "\u5238": -1133, "\u610F": 708, "\u3050": 159, "\u30D0": -254, "\u3082": -93, "\u3046": 325, "\u5165": -341, "\u79FB": -571, "\u30C8": -313, "\u4F53": -1085, "\u5F53": 401, "\u3005": 505, "\u5206": 173, "\u30A4": 171, "\u898B": -272, "\u6765": -122, "\u6599": 579, "\u5BFA": -461, "\u304E": 348, "\u624B": 179, "\u7D9A": -872, "\u3080": 458, "\u901A": -209, "\uFF0E": -184, "\u30BF": 244, "\u884C": -180, "\uFF1F": -192, "\u6C34": -154, "\u5148": -562, "\u90FD": 40, "\u4E00": -151, "\u3069": -41, "\u3070": 209, "\u30A2": -123, "\u5B66": 49, "\u6700": -116, "\u964D": 104, "\uFF4F": 95, "\u3078": -146, "\u4F55": -176, "\u52B9": 111, "\u30F3": -30, "\u3087": 90, "\u30FC": 4, "\u3000": -135, "\uFF08": -67, "\u7A2E": 42, "\uFF17": -101, "\u7684": -54, "\u6210": 46, "\u30E9": 29, "\uFF2E": 104, "\uFF11": -29, "\u6709": -37, "\u4E2D": 8, "\u5834": 4 }, "UW2": { "\u3002": -1401, "\u3001": -1370, "\u3092": -2403, "\u306B": -1207, "\u304C": -653, "\u306F": -1010, "\u304B": 18, "\u308C": 5, "\u3063": 269, "\u4EE5": 2215, "\u3066": -985, "\u3046": -489, "\u5168": 1762, "\u306E": -695, "\u305D": -534, "\u3067": -810, "\u3060": 523, "\u3086": 1668, "\u3055": 922, "\u304F": -812, "\u3081": 81, "\u3068": -751, "\u3082": -813, "\u3042": -233, "\u4E00": 1197, "\u3057": -49, "\u3093": 344, "\u304A": -711, "\u98DF": 981, "\u3054": -682, "\u304D": -556, "\u6BCE": 1671, "\u7D50": 1952, "\u5F7C": 754, "\u5C11": 676, "\u6700": 445, "\u307B": 1001, "\u308F": -118, "\u821E": 2257, "\u4E0A": -421, "\u3088": 577, "\u540C": 567, "\u308A": -479, "\u6A5F": 1341, "\u4EBA": 201, "\u610F": 531, "\u306A": -492, "\u300C": -482, "\u3070": -464, "\u95A2": 745, "\u4EAC": -377, "\u672C": 694, "\u3044": -79, "\u5927": 266, "\u76EE": -1004, "\u591A": 387, "\u7740": -477, "\u3064": -423, "\u30B1": -246, "\u632F": -1313, "\u3083": 283, "\u984C": 848, "\u624B": 663, "\u3072": -1353, "\u5BB9": 956, "\u6C34": 335, "\u6211": -1421, "\u308B": -207, "\u88CF": -1437, "\u30FC": 274, "\u305C": 567, "\u3071": 396, "\u540D": 341, "\u30D5": -264, "\u5BFA": -551, "\u8449": 787, "\u307E": -173, "\u5915": 826, "\u3076": 435, "\u7D76": 1181, "\u9AD8": 498, "\u4ECA": 276, "\u53CB": 494, "\u3089": -91, "\u30F3": 167, "\u305F": -109, "\u307F": -123, "\u6599": 70, "\u30A4": -122, "\u96FB": 315, "\u51B7": -664, "\u6253": 557, "\u30E1": -47, "\u4E16": -793, "\u30D1": 289, "\u5973": -487, "\uFF52": 990, "\u30D7": -195, "\u53D6": -360, "\uFF3E": 149, "\u5834": 190, "\u3053": 161, "\uFF0C": -306, "\u843D": -499, "\u9078": 313, "\u77E5": -361, "\u6975": 748, "\u5165": -160, "\u826F": 183, "\u30E9": -210, "\u30B9": 54, "\u904A": -310, "\u7528": -161, "\u5F0F": 93, "\u3005": 80, "\u8ECA": 124, "\u7ACB": -292, "\u4E0B": 154, "\u30EB": 31, "\u7518": 227, "\u5B9F": 97, "\u91CD": 159, "\u4E57": -115, "\u5F53": 94, "\uFF1A": 106, "\uFF3D": -50, "\uFF28": -84, "\u7684": 42, "\u76F8": 75, "\u4E09": 58, "\u3051": 16, "\u683C": 130, "\u6B63": 100, "\uFF3B": -16, "\u65E5": -25, "\u516C": -33, "\uFF14": 20, "\u30A3": -12, "\u308D": -4 }, "UW6": { "\u304C": 320, "\u3002": -307, "\u3066": -16, "\u3067": 219, "\u306F": 307, "\u3057": 581, "\u5165": 621, "\u3069": 828, "\u3042": -496, "\u7A7A": 1589, "\u306A": 218, "\u3059": -90, "\u3060": 95, "\u3081": 1415, "\u5238": 1711, "\u3055": 555, "\u3054": -283, "\u306E": 85, "\u6027": 2064, "\u3046": 35, "\u3084": 819, "\u308A": 578, "\u5834": -864, "\u306B": 245, "\u3064": 506, "\u308D": 352, "\u304D": -17, "\u3063": 301, "\u308B": 71, "\u304B": 231, "\u3044": 365, "\u3089": 65, "\u305D": -101, "\u3070": 527, "\u3051": -213, "\u30FC": -66, "\u3093": 324, "\u5F7C": -1272, "\u30AF": -346, "\u30EB": 207, "\uFF10": 626, "\u304F": 240, "\u3001": 180, "\u98DF": -284, "\u904A": -906, "\u5BFA": 611, "\u30FB": -639, "\uFF1A": 1217, "\u8A00": -361, "\u3086": 755, "\u614B": 221, "\u8FBC": 679, "\u3092": 85, "\u7684": 479, "\u53EF": 567, "\u30BF": 380, "\u6C17": 415, "\u601D": -111, "\u3071": 813, "\u307E": 170, "\u30E1": 239, "\u304A": -18, "\u30D4": 780, "\u30AB": 292, "\u9001": -896, "\u6765": -261, "\u904E": 846, "\u30A4": -244, "\u52B9": -725, "\u30B7": -130, "\u30B1": -388, "\u305A": -403, "\u3076": -248, "\u713C": 643, "\u308C": 119, "\u5E83": -150, "\u6A5F": 216, "\u308F": 60, "\u30F3": -64, "\u65B9": -235, "\uFF11": 33, "\u3053": 41, "\u5E97": 164, "\u5370": 360, "\u3073": 129, "\u56DE": -94, "\u304E": 280, "\u4E0D": 204, "\u3082": -18, "\u81EA": 123, "\u3080": 168, "\u30D1": -112, "\u5468": 275, "\u51FA": 68, "\u53C2": 114, "\u795E": 33, "\u7740": -20, "\u9593": -4, "\u7DF4": 8, "\u3083": -4 }, "UW1": { "\u306F": -273, "\u3046": 275, "\u307E": 128, "\u304A": 922, "\u3054": 890, "\u3068": 165, "\u304C": -385, "\u308B": -34, "\u305B": 210, "\u3044": -91, "\u3093": 289, "\u3042": -230, "\u3067": -127, "\u5B66": 209, "\u308A": -102, "\u6D6E": 1464, "\u306B": -344, "\u7518": -1424, "\u3064": 241, "\u30D0": 511, "\u306A": -220, "\u3066": 60, "\u6C34": 1969, "\u3001": -26, "\u96FB": 127, "\u5F53": -1038, "\u3075": 1899, "\u3089": 72, "\u308C": -44, "\u643A": -86, "\u3092": -176, "\u305D": 137, "\u3063": 194, "\u308D": 303, "\uFF3B": -601, "\u30FC": 113, "\u304B": -110, "\u9593": 572, "\u3061": 285, "\u3086": -428, "\u3057": -30, "\u5916": 301, "\u30D5": -155, "\u3082": -96, "\uFF10": 275, "\u7A7A": -524, "\u30B9": -251, "\u304F": 134, "\u81EA": 412, "\u3059": 151, "\u7528": 599, "\u3055": -164, "\u30C1": 183, "\u30B1": -196, "\u5165": 266, "\u30C4": 134, "\u56F0": 574, "\u5229": -371, "\u751F": 254, "\u6A5F": 318, "\u5B89": -400, "\u753B": -53, "\u5EAD": 619, "\u30A4": 211, "\uFF13": -258, "\u30E1": 96, "\u9762": -306, "\u4F55": 173, "\u3053": 64, "\u554F": 588, "\u65B9": 124, "\u767D": -248, "\u3058": -49, "\u793E": 225, "\u69D8": 365, "\u7530": 137, "\u56FD": -382, "\u8FBC": 42, "\u306E": -31, "\u9AD8": 170, "\u3002": -17, "\u30C3": 117, "\u3060": 63, "\u65B0": 178, "\uFF0E": -72, "\u30E9": 92, "\u4F7F": -79, "\u898B": 76, "\uFF1F": 113, "\u632F": 209, "\uFF01": 65, "\u64AE": -182, "\u5238": 79, "\u3056": 120, "\u5468": 137, "\u9577": 144, "\u3084": -42, "\u5C71": 114, "\u30D1": 63, "\u597D": 50, "\u30DA": 126, "\u7740": -16, "\u3081": -12, "\u307F": -8, "\u500B": -79, "\u4E0A": 12, "\u3048": -8, "\u9023": -29, "\u30DC": 16 }, "BW2": { "\u305F\u3068": 1475, "\u3068\u304A": -3875, "\u3068\u3044": 274, "\u3046\u3084": -2003, "\u306A\u3044": -2999, "\u306E\u307F": -2622, "\u3066\u3044": -1296, "\u3067\u3042": -3302, "\u306E\u3088": -838, "\u3067\u3059": -3035, "\u305F\u307E": -2116, "\u3044\u305F": -1737, "\u306E\u6A5F": -2082, "\u306E\u3067": -921, "\u3067\u304D": -2119, "\u306B\u306F": -1012, "\u306B\u3082": -1213, "\u306F\u3058": -2465, "\u3001\u305D": -1810, "\u304C\u3068": -1022, "\u304F\u306A": -1771, "\u3067\u3082": -1689, "\u306F\u306A": -331, "\u306E\u306F": -1100, "\u308B\u3068": -643, "\u3066\u304A": -2518, "\u306A\u3069": -1807, "\u5E2F\u96FB": -1351, "\u3067\u3057": -1856, "\u3001\u3068": -558, "\u307E\u305F": -2737, "\u305F\u3060": -2687, "\u304F\u3057": -1520, "\u308B\u3088": -944, "\u3044\u3082": 709, "\u3082\u3042": -935, "\u308A\u3082": -1014, "\u3066\u3042": -2039, "\u305F\u3044": -1183, "\u3066\u307F": -1524, "\u305D\u3053": -652, "\u3082\u3057": -490, "\u3066\u304F": -1248, "\u3069\u3082": -675, "\u3046\u3069": -1684, "\u306E\u524D": -820, "\u306E\u3060": -873, "\u306A\u304F": -956, "\u306B\u306A": -362, "\u4EBA\u304C": 1674, "\u3093\u3069": -722, "\u3082\u306E": -753, "\u3044\u307E": -421, "\u3068\u601D": 545, "\u308C\u305F": -890, "\u306F\u3044": -770, "\u3059\u3054": -1196, "\u304F\u3060": -1043, "\u30FB\u30FB": -1221, "\u3066\u304D": -1069, "\u304C\u3063": -1494, "\u306F\u3042": -1185, "\u3067\u306F": -1027, "\u3044\u3064": -659, "\u3068\u306A": 159, "\u6642\u9593": -681, "\u304C\u3057": 280, "\u306B\u3057": -458, "\u3046\u307E": -1227, "\u305F\u3061": -851, "\u306E\u304A": 458, "\u3070\u3044": -968, "\u3001\u306A": -145, "\u305F\u6642": 421, "\u306E\u3082": -489, "\u3068\u304B": -782, "\u3089\u306A": -64, "\u306A\u304B": -299, "\u306E\u304B": -278, "\u3067\u306A": -1003, "\u305F\u3082": 318, "\u3068\u3053": -337, "\u3082\u306A": -157, "\u3057\u305F": -402, "\u3082\u3064": -433, "\u3044\u3044": -357, "\u30FC\u30C9": -523, "\u3068\u304D": -230, "\u3082\u3061": -157, "\u305F\u3088": -393, "\u306F\u3067": 41, "\u3068\u3082": -359, "\u3044\u3057": -401, "\u3044\u3088": -494, "\u306F\u305A": -333, "\u3084\u3059": -524, "\u3068\u540C": -442, "\u3088\u3046": -175, "\u3044\u306A": -144, "\u30FC\u306A": 476, "\u308B\u306E": -197, "\u3057\u3044": -297, "\u3001\uFF11": -643, "\u3066\u3082": -151, "\u308A\u306A": 191, "\u304C\u3088": 195, "\u306E\uFF11": -539, "\u3046\u304B": -137, "\u3001\u6E05": -374, "\u4ECA\u65E5": -360, "\u3068\u3059": -127, "\u3064\u306E": 186, "\uFF3D\u4EAC": -152, "\u3046\u3044": -93, "\u306E\u304C": -101, "\u3044\u4EBA": 80, "\u3051\u3069": -126, "\u30FC\u3067": 83, "\u304C\u3061": 62, "\u306E\u65B9": 96, "\u3089\u3044": -54, "\u3067\u3044": -33, "\u3066\u305F": -37, "\u3069\u3053": -20 }, "BW1": { "\u306E\u307F": 3058, "\u4EE5\u4E0A": 1223, "\u307E\u307E": 2830, "\u304B\u3089": 1723, "\u305F\u307E": 2162, "\u308C\u305F": 1494, "\u767D\u3044": 3534, "\u3061\u304C": -2703, "\u3081\u3066": 969, "\u306A\u3044": 1045, "\u305D\u306E": -1066, "\u3067\u306F": -518, "\u3063\u305F": 166, "\u305F\u3068": -948, "\u3053\u3068": -1455, "\u304F\u308C": 922, "\u3068\u304B": 1688, "\u308B\u306E": -1593, "\u3046\u306A": 513, "\u305F\u306E": -1543, "\u306E\u306F": 1277, "\u3088\u308A": 1376, "\u3067\u3082": 667, "\u3068\u304C": -1393, "\u3057\u3044": 917, "\u304B\u3082": -1290, "\u30C3\u30AF": 631, "\u3088\u3046": -1620, "\u3093\u306A": 471, "\u3068\u3046": 1085, "\u3002\u30FB": 2456, "\u305F\u3089": 949, "\u6C17\u306B": -70, "\u3066\u3082": 623, "\u5229\u7528": 1018, "\u6700\u8FD1": 1515, "\u304C\u3068": -1545, "\u3053\u306E": -1039, "\u306B\u3082": 816, "\u3082\u3064": 1694, "\u304C\u3001": 1178, "\u308C\u3066": -1090, "\u308A\u304C": -96, "\u308A\u3068": 1198, "\u3093\u3067": -325, "\u307E\u3060": 1343, "\u306B\u306F": 588, "\u3082\u3046": 1414, "\u5C11\u3057": 1375, "\u3068\u304D": 1349, "\u3053\u305D": 1413, "\u308A\u3082": 1231, "\u308C\u3070": 936, "\u3051\u3069": 1014, "\u540C\u3058": 1427, "\u308B\u3068": 202, "\u305F\uFF0E": 1736, "\u307E\u308A": 1018, "\u3044\u3046": 138, "\u306F\u3001": 436, "\u3068\u3044": -725, "\u3044\u3001": -329, "\u306A\u3089": 1251, "\u601D\u3044": -1273, "\u3066\u3001": 702, "\u3042\u308B": 141, "\u3063\u3066": -196, "\u3067\u3001": 572, "\u306A\u305C": 1571, "\u308A\u306B": 356, "\u308A\u306E": -399, "\u307E\u308B": -429, "\u308C\u308B": 350, "\u305F\u3044": 242, "\u305D\u308C": 293, "\u3082\u3057": 1050, "\u306E\u9593": 1155, "\u306A\u3069": 1137, "\u3044\u306E": -369, "\u3046\u3084": -1096, "\u3060\u3051": 391, "\u3066\u3044": -522, "\u3068\u3067": 941, "\u304F\u3066": 680, "\u3093\u3060": 515, "\u30B1\u30FC": -781, "\u3044\u3067": -844, "\u666E\u6BB5": 655, "\u306E\u3082": 725, "\u601D\u3046": 57, "\u3048\u3070": 791, "\u304D\u306A": 687, "\u306B\u611F": -579, "\u3044\u305F": 310, "\u3001\u3068": 310, "\u3082\u306E": -538, "\u305F\u3060": 398, "\u306E\u304C": 482, "\u3066\u308B": 372, "\u5927\u5909": 401, "\u306E\u5F8C": 753, "\u308B\u4EBA": 626, "\u3057\u304B": 96, "\u3067\u304D": -364, "\u308B\u3067": 252, "\u307E\u3041": 444, "\u4F7F\u3044": -715, "\u3053\u3046": -591, "\u304B\u306A": -137, "\u8CB7\u3044": -698, "\uFF01\uFF01": 322, "\u3081\u308B": -429, "\u306B\u3001": 564, "\uFF10\u5186": 304, "\u3001\u4ECA": 302, "\u30FC\u30B8": 789, "\u3059\u308B": -159, "\u53CB\u9054": 625, "\u30AF\u30EB": 437, "\u308B\u306B": -442, "\u3084\u3089": 416, "\u3002\u3042": -619, "\u304D\u308A": 449, "\u3068\u3001": 245, "\u306E\u4E0A": 589, "\u305F\u3081": 524, "\u307E\u3067": 160, "\u304B\u3001": -255, "\u3046\u3061": 218, "\u306A\u304F": 149, "\u6765\u3066": 386, "\u304D\u305F": 236, "\u3079\u304D": 424, "\u3058\u3066": 254, "\u3071\u308A": 342, "\u3093\u3069": -303, "\u30FB\u30FB": 237, "\u3060\u306A": -302, "\u308D\u3046": -315, "\u73FE\u5728": 281, "\u3002\uFF13": 274, "\u304A\u5E97": 199, "\u3066\u305F": 147, "\u3059\u3089": 190, "\u3044\u3082": -216, "\u306B\u304B": 180, "\u3044\u306A": -98, "\u6C17\u304C": 115, "\u305F\u308A": 102, "\u308F\u3089": -71, "\u5B9F\u969B": 83, "\u3068\u306B": -20, "\u3082\u3001": 29, "\u304F\u3055": -24, "\u56FD\u4EBA": 24, "\u3046\u304B": 12, "\u306F\uFF11": 37, "\u3068\u306F": -4 }, "BW3": { "\u3046\u307E": 4971, "\u3082\u306E": 3479, "\u3042\u308A": 1053, "\u3044\u305F": 712, "\u3088\u3046": -404, "\u3044\u3046": 719, "\u4E0A\u304C": 1571, "\u307E\u3059": -943, "\u306F\u3058": 3173, "\u3068\u3046": -693, "\u305F\u307E": 1383, "\u3061\u304C": 1979, "\u3044\u3044": 1237, "\u304A\u308A": -2875, "\u306E\u307F": 3096, "\u53EF\u80FD": 402, "\u3061\u3083": -1016, "\u3053\u3068": 899, "\u305D\u3053": 647, "\u3067\u3082": 1643, "\u305D\u306E": 1844, "\u306A\u3063": -512, "\u3068\u3044": -1257, "\u3042\u308B": -339, "\u3053\u306E": 1656, "\u3068\u304D": 1917, "\u305D\u3046": -1050, "\u624B\u306B": 1242, "\u304B\u3089": -1223, "\u6301\u3063": 1098, "\u3067\u304D": 980, "\u304B\u3051": 2909, "\u3068\u3066": 2245, "\u306A\u308B": 481, "\u306F\u3057": 3011, "\u3061\u3087": 1986, "\u3057\u3066": 278, "\u3068\u304A": 678, "\u3082\u3089": 2538, "\u30E1\u30FC": 364, "\u3053\u308D": -190, "\u305D\u3057": -184, "\u304B\u304B": 1752, "\u3082\u3046": 2306, "\u3069\u3093": -397, "\u30FB\u30FB": -1499, "\u3088\u308A": -1463, "\u3057\u307E": -93, "\u3068\u3082": 1038, "\u3081\u3066": -1778, "\u3082\u3093": 1078, "\u3044\u308D": 832, "\u307E\u3041": 1442, "\u3060\u3051": -941, "\u3057\u305F": 813, "\u3055\u3093": -1566, "\u3046\u3061": 444, "\u304D\u3063": 727, "\u3088\u304F": 1082, "\u3059\u3054": 897, "\u4EE5\u4E0A": -807, "\u306A\u3044": 287, "\u305F\u3081": 686, "\u3053\u308C": 1075, "\u3069\u3046": 296, "\u306F\u307E": 1041, "\u3059\u3002": -152, "\u3068\u3063": 917, "\u6765\u305F": 408, "\u3057\u3083": 1283, "\u304B\u3063": -802, "\u305F\u3060": 1198, "\u3057\u3044": -519, "\u3059\u3050": 1094, "\u3082\u3068": 1264, "\u3084\u3063": 422, "\u304B\u3082": -670, "\u3081\u308B": -1304, "\u96FB\u8ECA": 121, "\u307E\u3057": -670, "\u3044\u304B": 480, "\u30D1\u30BD": 589, "\u308F\u304B": 691, "\u306A\u304F": 348, "\u3044\u308B": 556, "\u3068\u3053": 742, "\u4E2D\u3067": 414, "\u3055\u3044": -672, "\u307F\u305F": -739, "\u304B\u3064": 821, "\u3044\u3066": -372, "\u306A\u304B": 198, "\u306A\u3089": -160, "\u3060\u3044": 680, "\u3042\u3063": -93, "\u3082\u3063": 626, "\u307E\u305F": -479, "\u3044\u3084": 732, "\u8003\u3048": 429, "\u3046\u304B": -268, "\u3082\u3064": 666, "\u307E\u3060": 296, "\u304D\u307E": -514, "\u307E\u3067": -677, "\u304A\u8336": 627, "\u3053\u305D": -520, "\u3068\u308A": 539, "\u3044\u3089": 621, "\u3067\u3059": -239, "\u307E\u308A": -596, "\u51FA\u6765": -428, "\u3064\u306A": 437, "\u307B\u3057": -350, "\u305F\u3044": -238, "\u3053\u3053": 552, "\u306F\u3044": 272, "\u884C\u304D": -218, "\u697D\u3057": 128, "\u3057\u3001": -136, "\u3081\u3063": 260, "\u3093\u306F": 438, "\u51FA\u3066": 128, "\u3044\u307E": 85, "\u3064\u3051": 251, "\u307F\u304C": -374, "\u307E\u305A": 233, "\u3060\u308D": -357, "\u30EB\u30FC": 249, "\u304C\u3061": -345, "\u6599\u7406": 8, "\u796D\u308A": 214, "\u4F7F\u3063": 109, "\u3068\u601D": -161, "\u307C\u304F": 329, "\u30A4\u30F3": 203, "\u9006\u306B": 246, "\u3067\u3001": 119, "\u3069\u3053": 46, "\u305A\u3063": 121, "\u3064\u3044": 62, "\u307E\u305B": -58, "\u3042\u306E": 71, "\u904E\u304E": -95, "\u59CB\u3081": -28, "\u307E\u3063": -24, "\u7A0B\u5EA6": -24, "\u66F8\u3044": 12 }, "TW3": { "\u3068\u3044\u3046": 970, "\u3082\u306A\u304F": -1899, "\u3068\u3053\u308D": -1724, "\u3068\u3057\u3066": 655, "\u3001\u3042\u308B": -2756, "\u3066\u3044\u308B": -1375, "\u306F\u306A\u3044": -473, "\u306E\u307E\u307E": -961, "\u3059\u3053\u3068": 710, "\u3066\u3057\u307E": -1361, "\u3089\u306A\u3044": -1075, "\u306F\u305D\u308C": -1539, "\u3066\u3044\u304F": -289, "\u3068\u3057\u305F": -1169, "\u3068\u8A00\u3063": -1166, "\u3066\u3044\u3046": 769, "\u3067\u3042\u308B": -749, "\u3001\u306A\u3093": -293, "\u308B\u3053\u3068": -49, "\u3067\u3042\u3063": -812, "\u3068\u306A\u3063": 229, "\u306A\u3093\u3067": 574, "\u3044\u3068\u3053": 75, "\u3082\u3044\u3044": -408, "\u3082\u3042\u308A": -183, "\u3067\u3042\u308A": -535, "\u3046\u3044\u3046": -120, "\u3070\u3044\u3044": -137, "\u306B\u3057\u3066": -37, "\u3066\u3044\u305F": -66, "\u308B\u3068\u304D": 108, "\u3067\u3057\u305F": -54, "\u3046\u3053\u3068": -4 }, "TW4": { "\u3068\u3053\u308D": 2195, "\u3088\u3046\u306B": -1673, "\u304F\u3089\u3044": 1598, "\u3057\u305F\u3002": -1569, "\u3042\u308A\u307E": -1767, "\u304B\u306A\u308A": 2848, "\u3067\u304D\u305F": 2069, "\u307E\u3059\u3002": -1366, "\u3088\u3046\u306A": -2232, "\u3053\u3068\u304C": -824, "\u3044\u305F\u3002": 1066, "\u3060\u304B\u3089": 1092, "\u3046\u3069\u3093": 1258, "\u305D\u3046\u3067": -925, "\u3053\u3068\u306B": -856, "\u3082\u3061\u308D": 1697, "\u3057\u304B\u3057": 1383, "\u3053\u3068\u3067": 380, "\u306A\u3044\u3002": 191, "\u3053\u3068\u3082": -588, "\u3042\u308A\u3001": -595, "\u3061\u306A\u307F": 1130, "\u3060\u3063\u305F": -1150, "\u3059\u308B\u3068": 406, "\u3053\u3068\u3002": 683, "\u3067\u3082\u3001": 628, "\u3044\u308D\u3044": 590, "\u3044\u304B\u306A": 651, "\u30E1\u30FC\u30EB": 256, "\u3088\u3046\u3067": -896, "\u306A\u304B\u306A": 669, "\u306A\u304F\u306A": 253, "\u3057\u306A\u3044": 438, "\u30BD\u30D5\u30C8": 512, "\u3053\u3068\u3092": 255, "\u308F\u3051\u3067": -234, "\u307E\u3063\u3066": -824, "\u306A\u3093\u3068": 481, "\u3059\u3054\u3044": 314, "\u3057\u3066\u304D": 278, "\u3044\u3048\u3070": 397, "\u3089\u3057\u3044": 424, "\u3057\u304B\u3082": 494, "\u3053\u3068\u306F": -232, "\u3042\u305F\u308A": -390, "\u98DF\u3079\u3066": 140, "\u3057\u3066\u3082": -174, "\u3067\u304D\u307E": 52, "\u306A\u3093\u304B": 116, "\u306A\u3093\u3066": -102, "\u3044\u304F\u3089": 170, "\u89B3\u5149\u5BA2": -75, "\u3044\u3044\u3068": -63, "\u3044\u308B\u3002": 37, "\u3061\u3087\u3063": 20, "\u305D\u3046\u3044": 16 }, "TW2": { "\u6C17\u306B\u5165": -4086, "\u3067\u306F\u306A": -1086, "\u3068\u304C\u3042": -1130, "\u304B\u3082\u3057": -463, "\u3093\u3067\u3044": -2306, "\u305D\u3082\u305D": -1441, "\u3046\u306B\u306A": -803, "\u3057\u3066\u3044": -406, "\u3067\u3082\u3042": -936, "\u3063\u305F\u3053": -267, "\u3057\u304B\u306A": 440, "\u3068\u3082\u3042": -757, "\u304B\u3089\u306A": -477, "\u3048\u308B\u3053": -436, "\u3067\u3082\u306A": -207, "\u3068\u306F\u3044": -59, "\u3048\u3066\u3044": 50, "\u3057\u3066\u304A": -37 }, "TW1": { "\u3068\u3044\u3046": 545, "\u3066\u3044\u304F": 1686, "\u3088\u3046\u306A": 1298, "\u30FB\u30FB\u30FB": 1104, "\u6301\u3063\u3066": -1851, "\u3057\u304B\u3057": 2079, "\u3066\u304F\u308C": 1444, "\u306E\u307E\u307E": 1162, "\u304B\u306A\u308A": 2083, "\u3084\u306F\u308A": 1728, "\u306A\u304C\u3089": 1223, "\u3087\u3046\u3069": 1174, "\u3082\u306A\u304F": 947, "\u601D\u3063\u305F": 811, "\u3068\u3093\u3069": 1250, "\u3001\u305D\u308C": 412, "\u306A\u3044\u3001": -1039, "\u306A\u3093\u3068": -691, "\u3092\u5229\u7528": -1017, "\u304C\u3063\u3066": -418, "\u3072\u3068\u3064": 732, "\u3068\u601D\u3046": 251, "\u307E\u3057\u305F": -268, "\u3067\u3042\u308B": 267, "\u304B\u306A\u304B": 698, "\u305D\u3057\u3066": 556, "\u3063\u304B\u308A": 599, "\u3002\u305F\u3060": 676, "\u30EA\u30FC\u30B0": 309, "\u304D\u306A\u3044": 141, "\u306F\u305D\u306E": -68, "\u3061\u308D\u3093": 196, "\u304F\u3055\u3093": 225, "\u308B\u3053\u3068": -271, "\u3060\u3063\u3066": 164, "\u306A\u3063\u3066": -86, "\u3059\u3054\u304F": 163, "\u308D\u3044\u308D": 54, "\u3002\u305D\u306E": 16, "\u304F\u3089\u3044": 4 } };
// module/dom-browser.js
var parseFromString = (html) => {
return new window.DOMParser().parseFromString(html, "text/html");
};
// module/parser.js
var Parser = class {
/**
* Constructs a BudouX parser.
* @param model A model data.
*/
constructor(model2) {
this.model = new Map(Object.entries(model2).map(([k, v]) => [k, new Map(Object.entries(v))]));
this.baseScore = -0.5 * [...this.model.values()].map((group) => [...group.values()]).flat().reduce((prev, curr) => prev + curr, 0);
}
/**
* Parses the input sentence and returns a list of semantic chunks.
*
* @param sentence An input sentence.
* @return The retrieved chunks.
*/
parse(sentence) {
if (sentence === "")
return [];
const boundaries = this.parseBoundaries(sentence);
const result = [];
let start = 0;
for (const boundary of boundaries) {
result.push(sentence.slice(start, boundary));
start = boundary;
}
result.push(sentence.slice(start));
return result;
}
/**
* Parses the input sentence and returns a list of boundaries.
*
* @param sentence An input sentence.
* @return The list of boundaries.
*/
parseBoundaries(sentence) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
const result = [];
for (let i = 1; i < sentence.length; i++) {
let score = this.baseScore;
score += ((_a = this.model.get("UW1")) === null || _a === void 0 ? void 0 : _a.get(sentence.substring(i - 3, i - 2))) || 0;
score += ((_b = this.model.get("UW2")) === null || _b === void 0 ? void 0 : _b.get(sentence.substring(i - 2, i - 1))) || 0;
score += ((_c = this.model.get("UW3")) === null || _c === void 0 ? void 0 : _c.get(sentence.substring(i - 1, i))) || 0;
score += ((_d = this.model.get("UW4")) === null || _d === void 0 ? void 0 : _d.get(sentence.substring(i, i + 1))) || 0;
score += ((_e = this.model.get("UW5")) === null || _e === void 0 ? void 0 : _e.get(sentence.substring(i + 1, i + 2))) || 0;
score += ((_f = this.model.get("UW6")) === null || _f === void 0 ? void 0 : _f.get(sentence.substring(i + 2, i + 3))) || 0;
score += ((_g = this.model.get("BW1")) === null || _g === void 0 ? void 0 : _g.get(sentence.substring(i - 2, i))) || 0;
score += ((_h = this.model.get("BW2")) === null || _h === void 0 ? void 0 : _h.get(sentence.substring(i - 1, i + 1))) || 0;
score += ((_j = this.model.get("BW3")) === null || _j === void 0 ? void 0 : _j.get(sentence.substring(i, i + 2))) || 0;
score += ((_k = this.model.get("TW1")) === null || _k === void 0 ? void 0 : _k.get(sentence.substring(i - 3, i))) || 0;
score += ((_l = this.model.get("TW2")) === null || _l === void 0 ? void 0 : _l.get(sentence.substring(i - 2, i + 1))) || 0;
score += ((_m = this.model.get("TW3")) === null || _m === void 0 ? void 0 : _m.get(sentence.substring(i - 1, i + 2))) || 0;
score += ((_o = this.model.get("TW4")) === null || _o === void 0 ? void 0 : _o.get(sentence.substring(i, i + 3))) || 0;
if (score > 0)
result.push(i);
}
return result;
}
};
// module/html_processor.js
var assert = console.assert;
var ZWSP_CODEPOINT = 8203;
var ZWSP = String.fromCharCode(ZWSP_CODEPOINT);
var NodeType = {
ELEMENT_NODE: 1,
TEXT_NODE: 3
};
var DomAction = {
Inline: 0,
// An inline content, becomes a part of a paragraph.
Block: 1,
// A nested paragraph.
Skip: 2,
// Skip the content. The content before and after are connected.
Break: 3,
// A forced break. The content before and after become paragraphs.
NoBreak: 4,
// The content provides context, but it's not breakable.
BreakOpportunity: 5
// Force a break opportunity.
};
var domActions = {
// Hidden elements
// https://html.spec.whatwg.org/multipage/rendering.html#hidden-elements
AREA: DomAction.Skip,
BASE: DomAction.Skip,
BASEFONT: DomAction.Skip,
DATALIST: DomAction.Skip,
HEAD: DomAction.Skip,
LINK: DomAction.Skip,
META: DomAction.Skip,
NOEMBED: DomAction.Skip,
NOFRAMES: DomAction.Skip,
PARAM: DomAction.Skip,
RP: DomAction.Skip,
SCRIPT: DomAction.Skip,
STYLE: DomAction.Skip,
TEMPLATE: DomAction.Skip,
TITLE: DomAction.Skip,
NOSCRIPT: DomAction.Skip,
// Flow content
// https://html.spec.whatwg.org/multipage/rendering.html#flow-content-3
HR: DomAction.Break,
// Disable if `white-space: pre`.
LISTING: DomAction.Skip,
PLAINTEXT: DomAction.Skip,
PRE: DomAction.Skip,
XMP: DomAction.Skip,
// Phrasing content
// https://html.spec.whatwg.org/multipage/rendering.html#phrasing-content-3
BR: DomAction.Break,
RT: DomAction.Skip,
WBR: DomAction.BreakOpportunity,
// Form controls
// https://html.spec.whatwg.org/multipage/rendering.html#form-controls
INPUT: DomAction.Skip,
SELECT: DomAction.Skip,
BUTTON: DomAction.Skip,
TEXTAREA: DomAction.Skip,
// Other elements where the phrase-based line breaking should be disabled.
// https://github.com/google/budoux/blob/main/budoux/skip_nodes.json
ABBR: DomAction.Skip,
CODE: DomAction.Skip,
IFRAME: DomAction.Skip,
TIME: DomAction.Skip,
VAR: DomAction.Skip,
// Deprecated, but supported in all browsers.
// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/nobr
NOBR: DomAction.NoBreak
};
var defaultBlockElements = /* @__PURE__ */ new Set([
// 15.3.2 The page
"HTML",
"BODY",
// 15.3.3 Flow content
"ADDRESS",
"BLOCKQUOTE",
"CENTER",
"DIALOG",
"DIV",
"FIGURE",
"FIGCAPTION",
"FOOTER",
"FORM",
"HEADER",
"LEGEND",
"LISTING",
"MAIN",
"P",
// 15.3.6 Sections and headings
"ARTICLE",
"ASIDE",
"H1",
"H2",
"H3",
"H4",
"H5",
"H6",
"HGROUP",
"NAV",
"SECTION",
// 15.3.7 Lists
"DIR",
"DD",
"DL",
"DT",
"MENU",
"OL",
"UL",
"LI",
// 15.3.8 Tables
"TABLE",
"CAPTION",
"COL",
"TR",
"TD",
"TH",
// 15.3.12 The fieldset and legend elements
"FIELDSET",
// 15.5.4 The details and summary elements
"DETAILS",
"SUMMARY",
// 15.5.12 The marquee element
"MARQUEE"
]);
function actionForDisplay(display) {
if (display === "inline")
return DomAction.Inline;
if (display === "block")
return DomAction.Block;
if (display.startsWith("ruby")) {
if (display === "ruby-text")
return DomAction.Skip;
return DomAction.Inline;
}
return DomAction.Block;
}
function actionForElement(element) {
const nodeName = element.nodeName;
const action = domActions[nodeName];
if (action !== void 0)
return action;
if (typeof globalThis.getComputedStyle === "function") {
const style = globalThis.getComputedStyle(element);
switch (style.whiteSpace) {
case "nowrap":
case "pre":
return DomAction.NoBreak;
}
const display = style.display;
if (display)
return actionForDisplay(display);
}
return defaultBlockElements.has(nodeName) ? DomAction.Block : DomAction.Inline;
}
var applyWrapStyle = (element) => {
element.style.wordBreak = "keep-all";
element.style.overflowWrap = "anywhere";
};
var NodeOrText = class _NodeOrText {
constructor(nodeOrText) {
this.chunks = [];
this.hasBreakOpportunityAfter = false;
this.nodeOrText = nodeOrText;
}
static isString(value) {
return typeof value === "string";
}
get canSplit() {
return !_NodeOrText.isString(this.nodeOrText);
}
get text() {
return _NodeOrText.isString(this.nodeOrText) ? this.nodeOrText : this.nodeOrText.nodeValue;
}
get length() {
var _a, _b;
return (_b = (_a = this.text) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
}
/**
* Split the {@link Text} in the same way as the {@link chunks}.
* Joining all {@link chunks} must be equal to {@link text}.
*/
split(separator) {
const chunks = this.chunks;
assert(chunks.length === 0 || chunks.join("") === this.text);
if (chunks.length <= 1)
return;
if (_NodeOrText.isString(this.nodeOrText))
return;
const node = this.nodeOrText;
if (typeof separator === "string") {
node.nodeValue = chunks.join(separator);
return;
}
const document = node.ownerDocument;
let nodes = [];
for (const chunk of chunks) {
if (chunk)
nodes.push(document.createTextNode(chunk));
nodes.push(null);
}
nodes.pop();
nodes = nodes.map((n) => n ? n : separator.cloneNode(true));
node.replaceWith(...nodes);
}
};
var NodeOrTextForTesting = class extends NodeOrText {
};
var Paragraph = class {
constructor(element) {
this.nodes = [];
this.element = element;
}
isEmpty() {
return this.nodes.length === 0;
}
get text() {
return this.nodes.map((node) => node.text).join("");
}
get lastNode() {
return this.nodes.length ? this.nodes[this.nodes.length - 1] : void 0;
}
setHasBreakOpportunityAfter() {
const lastNode = this.lastNode;
if (lastNode)
lastNode.hasBreakOpportunityAfter = true;
}
/**
* @return Indices of forced break opportunities in the source.
* They can be created by `<wbr>` tag or `​`.
*/
getForcedOpportunities() {
const opportunities = [];
let len = 0;
for (const node of this.nodes) {
if (node.canSplit) {
const text = node.text;
if (text) {
for (let i = 0; i < text.length; ++i) {
if (text.charCodeAt(i) === ZWSP_CODEPOINT) {
opportunities.push(len + i + 1);
}
}
}
}
len += node.length;
if (node.hasBreakOpportunityAfter) {
opportunities.push(len);
}
}
return opportunities;
}
/**
* @return Filtered {@param boundaries} by excluding
* {@link getForcedOpportunities} if it's not empty.
* Otherwise {@param boundaries}.
*/
excludeForcedOpportunities(boundaries) {
const forcedOpportunities = this.getForcedOpportunities();
if (!forcedOpportunities.length)
return boundaries;
const set = new Set(forcedOpportunities);
return boundaries.filter((i) => !set.has(i));
}
};
var HTMLProcessor = class {
/**
* @param parser A BudouX {@link Parser} to compute semantic line breaks.
*/
constructor(parser2, options) {
this.separator = ZWSP;
this.parser_ = parser2;
if (options !== void 0) {
if (options.className !== void 0)
this.className = options.className;
if (options.separator !== void 0)
this.separator = options.separator;
}
}
/**
* Checks if the given element has a text node in its children.
*
* @param ele An element to be checked.
* @return Whether the element has a child text node.
*/
static hasChildTextNode(ele) {
for (const child of ele.childNodes) {
if (child.nodeType === NodeType.TEXT_NODE)
return true;
}
return false;
}
/**
* Applies markups for semantic line breaks to the given HTML element.
*
* It breaks descendant nodes into paragraphs,
* and applies the BudouX to each paragraph.
* @param element The input element.
*/
applyToElement(element) {
for (const block of this.getBlocks(element)) {
assert(!block.isEmpty());
this.applyToParagraph(block);
}
}
/**
* Find paragraphs from a given HTML element.
* @param element The root element to find paragraphs.
* @param parent The parent {@link Paragraph} if any.
* @return A list of {@link Paragraph}s.
*/
*getBlocks(element, parent) {
assert(element.nodeType === NodeType.ELEMENT_NODE);
if (this.className && element.classList.contains(this.className))
return;
const action = actionForElement(element);
if (action === DomAction.Skip)
return;
if (action === DomAction.Break) {
if (parent && !parent.isEmpty()) {
parent.setHasBreakOpportunityAfter();
yield parent;
parent.nodes = [];
}
assert(!element.firstChild);
return;
}
if (action === DomAction.BreakOpportunity) {
if (parent)
parent.setHasBreakOpportunityAfter();
return;
}
assert(action === DomAction.Block || action === DomAction.Inline || action === DomAction.NoBreak);
const isNewBlock = !parent || action === DomAction.Block;
const block = isNewBlock ? new Paragraph(element) : parent;
for (const child of element.childNodes) {
switch (child.nodeType) {
case NodeType.ELEMENT_NODE:
for (const childBlock of this.getBlocks(child, block))
yield childBlock;
break;
case NodeType.TEXT_NODE:
if (action === DomAction.NoBreak) {
const text = child.nodeValue;
if (text) {
block.nodes.push(new NodeOrText(text));
}
break;
}
block.nodes.push(new NodeOrText(child));
break;
}
}
if (isNewBlock && !block.isEmpty())
yield block;
}
/**
* Apply the BudouX to the given {@link Paragraph}.
* @param paragraph The {@link Paragraph} to apply.
*/
applyToParagraph(paragraph) {
assert(paragraph.nodes.length > 0);
if (!paragraph.nodes.some((node) => node.canSplit))
return;
const text = paragraph.text;
if (/^\s*$/.test(text))
return;
const boundaries = this.parser_.parseBoundaries(text);
if (boundaries.length <= 0)
return;
assert(boundaries[0] > 0);
assert(boundaries.every((x, i) => i === 0 || x > boundaries[i - 1]));
assert(boundaries[boundaries.length - 1] < text.length);
const adjustedBoundaries = paragraph.excludeForcedOpportunities(boundaries);
adjustedBoundaries.push(text.length + 1);
this.splitNodes(paragraph.nodes, adjustedBoundaries);
this.applyBlockStyle(paragraph.element);
}
/**
* Split {@link NodeOrText} at the specified boundaries.
* @param nodes A list of {@link NodeOrText}.
* @param boundaries A list of indices of the text to split at.
*/
splitNodes(nodes, boundaries) {
var _a;
assert(boundaries.length > 0);
assert(boundaries.every((x, i) => i === 0 || x > boundaries[i - 1]));
const textLen = nodes.reduce((sum, node) => sum + node.length, 0);
assert(boundaries[boundaries.length - 1] > textLen);
let boundary_index = 0;
let boundary = boundaries[0];
assert(boundary > 0);
let nodeStart = 0;
let lastNode = null;
for (const node of nodes) {
assert(boundary >= nodeStart);
assert(node.chunks.length === 0);
const nodeText = node.text;
if (!nodeText)
continue;
const nodeLength = nodeText.length;
const nodeEnd = nodeStart + nodeLength;
assert(!lastNode || lastNode.canSplit);
if (!node.canSplit) {
if (lastNode && boundary === nodeStart) {
if (lastNode.chunks.length === 0)
lastNode.chunks.push((_a = lastNode.text) !== null && _a !== void 0 ? _a : "");
lastNode.chunks.push("");
}
while (boundary < nodeEnd) {
boundary = boundaries[++boundary_index];
}
lastNode = null;
nodeStart = nodeEnd;
continue;
}
lastNode = node;
if (boundary >= nodeEnd) {
nodeStart = nodeEnd;
continue;
}
const chunks = node.chunks;
let chunkStartInNode = 0;
while (boundary < nodeEnd) {
const boundaryInNode = boundary - nodeStart;
assert(boundaryInNode >= chunkStartInNode);
chunks.push(nodeText.slice(chunkStartInNode, boundaryInNode));
chunkStartInNode = boundaryInNode;
boundary = boundaries[++boundary_index];
}
assert(chunkStartInNode < nodeLength);
chunks.push(nodeText.slice(chunkStartInNode));
nodeStart = nodeEnd;
}
assert(nodeStart === textLen);
assert(boundary_index < boundaries.length);
assert(boundaries[boundary_index] >= textLen);
for (const node of nodes) {
node.split(this.separator);
}
}
/**
* Applies the block style to the given element.
* @param element The element to apply the block style.
*/
applyBlockStyle(element) {
if (this.className) {
element.classList.add(this.className);
return;
}
applyWrapStyle(element);
}
};
var HTMLProcessingParser = class extends Parser {
constructor(model2, htmlProcessorOptions = {
separator: ZWSP
}) {
super(model2);
this.htmlProcessor = new HTMLProcessor(this, htmlProcessorOptions);
}
/**
* Applies markups for semantic line breaks to the given HTML element.
* @param parentElement The input element.
*/
applyToElement(parentElement) {
this.htmlProcessor.applyToElement(parentElement);
}
/**
* Translates the given HTML string to another HTML string with markups
* for semantic line breaks.
* @param html An input html string.
* @return The translated HTML string.
*/
translateHTMLString(html) {
if (html === "")
return html;
const doc = parseFromString(html);
if (HTMLProcessor.hasChildTextNode(doc.body)) {
const wrapper = doc.createElement("span");
wrapper.append(...doc.body.childNodes);
doc.body.append(wrapper);
}
this.applyToElement(doc.body.childNodes[0]);
return doc.body.innerHTML;
}
};
// module/index.js
var loadDefaultJapaneseParser = () => {
return new HTMLProcessingParser(model);
};
// module/tests/testutils-browser.js
var setInnerHtml = (element, html) => {
element.innerHTML = html;
};
var createDocument = () => {
return window.document;
};
var isBrowser = true;
// module/tests/test_html_processor.js
var parser = loadDefaultJapaneseParser();
var MockHTMLProcessorBase = class extends HTMLProcessor {
constructor(options) {
super(parser, options);
}
};
function isEqualNodeWithStyleNormalized(a, b) {
if (a.nodeType !== b.nodeType)
return false;
if (a.nodeType === 3) {
return a.nodeValue === b.nodeValue;
}
if (a.nodeName !== b.nodeName)
return false;
if (a.attributes.length !== b.attributes.length)
return false;
for (let i = 0; i < a.attributes.length; i++) {
const attribName = a.attributes[i].name;
const aAttribValue = a.getAttribute(attribName);
const bAttribValue = b.getAttribute(attribName);
if (attribName === "style") {
const aNormalizedStyle = aAttribValue === null || aAttribValue === void 0 ? void 0 : aAttribValue.replace(/\s+/g, "").replace(/;$/, "");
const bNormalizedStyle = bAttribValue === null || bAttribValue === void 0 ? void 0 : bAttribValue.replace(/\s+/g, "").replace(/;$/, "");
if (aNormalizedStyle !== bNormalizedStyle)
return false;
} else {
if (aAttribValue !== bAttribValue)
return false;
}
}
if (a.childNodes.length !== b.childNodes.length)
return false;
for (let i = 0; i < a.childNodes.length; i++) {
if (!isEqualNodeWithStyleNormalized(a.childNodes[i], b.childNodes[i])) {
return false;
}
}
return true;
}
function getBlocks(html) {
const document = createDocument();
setInnerHtml(document.body, html);
const processor = new MockHTMLProcessorBase();
return processor.getBlocks(document.body);
}
describe("HTMLProcessor.applyToElement", () => {
const document = createDocument();
const wbr = document.createElement("wbr");
function apply(html, separator) {
setInnerHtml(document.body, html);
const processor = new MockHTMLProcessorBase({
separator,
className: "applied"
});
processor.applyToElement(document.body);
return document.body.innerHTML;
}
for (const test of [
{
in: "<div>\u6674\u308C</div>",
out: "<div>\u6674\u308C</div>",
browserOnly: false
},
{
in: "<div>\u4ECA\u65E5\u306F\u6674\u308C\u3067\u3059</div>",
out: '<div class="applied">\u4ECA\u65E5\u306F|\u6674\u308C\u3067\u3059</div>',
browserOnly: false
},
{
in: "<div><span>\u4ECA\u65E5\u306F</span>\u6674\u308C\u3067\u3059</div>",
out: '<div class="applied"><span>\u4ECA\u65E5\u306F</span>|\u6674\u308C\u3067\u3059</div>',
browserOnly: false
},
{
in: "<div><span>\u4ECA\u65E5\u306F\u6674\u308C</span>\u3067\u3059</div>",
out: '<div class="applied"><span>\u4ECA\u65E5\u306F|\u6674\u308C</span>\u3067\u3059</div>',
browserOnly: false
},
{
in: "<code>\u4ECA\u65E5\u306F\u6674\u308C\u3067\u3059</code>",
out: "<code>\u4ECA\u65E5\u306F\u6674\u308C\u3067\u3059</code>",
browserOnly: false
},
{
in: "<div>\u4ECA\u65E5\u306F<code>code</code>\u6674\u308C\u3067\u3059</div>",
out: '<div class="applied">\u4ECA\u65E5\u306F<code>code</code>|\u6674\u308C\u3067\u3059</div>',
browserOnly: false
},
{
in: "<div>\u4ECA\u65E5\u306F\u6674\u308C\u3001\u4ECA\u65E5\u306F\u6674\u308C</div>",
out: '<div class="applied">\u4ECA\u65E5\u306F|\u6674\u308C\u3001|\u4ECA\u65E5\u306F|\u6674\u308C</div>',
browserOnly: false
},
{
in: "<div>\u4ECA\u65E5\u306F<nobr>\u6674\u308C\u3001\u4ECA\u65E5\u306F</nobr>\u6674\u308C</div>",
out: '<div class="applied">\u4ECA\u65E5\u306F|<nobr>\u6674\u308C\u3001\u4ECA\u65E5\u306F</nobr>|\u6674\u308C</div>',
browserOnly: true
},
{
in: '<div>\u4ECA\u65E5\u306F<span style="white-space: nowrap">\u6674\u308C\u3001\u4ECA\u65E5\u306F</span>\u6674\u308C</div>',
out: '<div class="applied">\u4ECA\u65E5\u306F|<span style="white-space: nowrap">\u6674\u308C\u3001\u4ECA\u65E5\u306F</span>|\u6674\u308C</div>',
browserOnly: true
}
]) {
it(test.in, () => {
const out = test.out.replace(/\|/g, "<wbr>");
if (test.browserOnly && !isBrowser)
return;
const result = apply(test.in, wbr);
const isEqual = isEqualNodeWithStyleNormalized(parseFromString(re