@lenml/tokenizer-gemini/dist/main.mjs.map

gemini tokenizer for NodeJS/Browser

{"version":3,"sources":["../models/tokenizer.json","../models/tokenizer_config.json","../src/data.ts","../src/main.ts"],"sourcesContent":["{\n  \"version\": \"1.0\",\n  \"truncation\": null,\n  \"padding\": null,\n  \"added_tokens\": [\n    {\n      \"id\": 0,\n      \"content\": \"<pad>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": true\n    },\n    {\n      \"id\": 1,\n      \"content\": \"<eos>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": true\n    },\n    {\n      \"id\": 2,\n      \"content\": \"<bos>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": true\n    },\n    {\n      \"id\": 3,\n      \"content\": \"<unk>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": true\n    },\n    {\n      \"id\": 4,\n      \"content\": \"<mask>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 5,\n      \"content\": \"<2mass>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 6,\n      \"content\": \"[@BOS@]\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 7,\n      \"content\": \"<unused0>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 8,\n      \"content\": \"<unused1>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 9,\n      \"content\": \"<unused2>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 10,\n      \"content\": \"<unused3>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 11,\n      \"content\": \"<unused4>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 12,\n      \"content\": \"<unused5>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 13,\n      \"content\": \"<unused6>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 14,\n      \"content\": \"<unused7>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 15,\n      \"content\": \"<unused8>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 16,\n      \"content\": \"<unused9>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 17,\n      \"content\": \"<unused10>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 18,\n      \"content\": \"<unused11>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 19,\n      \"content\": \"<unused12>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 20,\n      \"content\": \"<unused13>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 21,\n      \"content\": \"<unused14>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 22,\n      \"content\": \"<unused15>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 23,\n      \"content\": \"<unused16>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 24,\n      \"content\": \"<unused17>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 25,\n      \"content\": \"<unused18>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 26,\n      \"content\": \"<unused19>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 27,\n      \"content\": \"<unused20>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 28,\n      \"content\": \"<unused21>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 29,\n      \"content\": \"<unused22>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 30,\n      \"content\": \"<unused23>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 31,\n      \"content\": \"<unused24>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 32,\n      \"content\": \"<unused25>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 33,\n      \"content\": \"<unused26>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 34,\n      \"content\": \"<unused27>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 35,\n      \"content\": \"<unused28>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 36,\n      \"content\": \"<unused29>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 37,\n      \"content\": \"<unused30>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 38,\n      \"content\": \"<unused31>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 39,\n      \"content\": \"<unused32>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 40,\n      \"content\": \"<unused33>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 41,\n      \"content\": \"<unused34>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 42,\n      \"content\": \"<unused35>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 43,\n      \"content\": \"<unused36>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 44,\n      \"content\": \"<unused37>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 45,\n      \"content\": \"<unused38>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 46,\n      \"content\": \"<unused39>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 47,\n      \"content\": \"<unused40>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 48,\n      \"content\": \"<unused41>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 49,\n      \"content\": \"<unused42>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 50,\n      \"content\": \"<unused43>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 51,\n      \"content\": \"<unused44>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 52,\n      \"content\": \"<unused45>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 53,\n      \"content\": \"<unused46>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 54,\n      \"content\": \"<unused47>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 55,\n      \"content\": \"<unused48>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 56,\n      \"content\": \"<unused49>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 57,\n      \"content\": \"<unused50>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 58,\n      \"content\": \"<unused51>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 59,\n      \"content\": \"<unused52>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 60,\n      \"content\": \"<unused53>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 61,\n      \"content\": \"<unused54>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 62,\n      \"content\": \"<unused55>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 63,\n      \"content\": \"<unused56>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 64,\n      \"content\": \"<unused57>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 65,\n      \"content\": \"<unused58>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 66,\n      \"content\": \"<unused59>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 67,\n      \"content\": \"<unused60>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 68,\n      \"content\": \"<unused61>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 69,\n      \"content\": \"<unused62>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 70,\n      \"content\": \"<unused63>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 71,\n      \"content\": \"<unused64>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 72,\n      \"content\": \"<unused65>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 73,\n      \"content\": \"<unused66>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 74,\n      \"content\": \"<unused67>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 75,\n      \"content\": \"<unused68>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 76,\n      \"content\": \"<unused69>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 77,\n      \"content\": \"<unused70>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 78,\n      \"content\": \"<unused71>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 79,\n      \"content\": \"<unused72>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 80,\n      \"content\": \"<unused73>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 81,\n      \"content\": \"<unused74>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 82,\n      \"content\": \"<unused75>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 83,\n      \"content\": \"<unused76>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 84,\n      \"content\": \"<unused77>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 85,\n      \"content\": \"<unused78>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 86,\n      \"content\": \"<unused79>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 87,\n      \"content\": \"<unused80>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 88,\n      \"content\": \"<unused81>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 89,\n      \"content\": \"<unused82>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 90,\n      \"content\": \"<unused83>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 91,\n      \"content\": \"<unused84>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 92,\n      \"content\": \"<unused85>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 93,\n      \"content\": \"<unused86>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 94,\n      \"content\": \"<unused87>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 95,\n      \"content\": \"<unused88>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 96,\n      \"content\": \"<unused89>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 97,\n      \"content\": \"<unused90>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 98,\n      \"content\": \"<unused91>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 99,\n      \"content\": \"<unused92>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 100,\n      \"content\": \"<unused93>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 101,\n      \"content\": \"<unused94>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 102,\n      \"content\": \"<unused95>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 103,\n      \"content\": \"<unused96>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 104,\n      \"content\": \"<unused97>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 105,\n      \"content\": \"<unused98>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 106,\n      \"content\": \"<start_of_turn>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": true\n    },\n    {\n      \"id\": 107,\n      \"content\": \"<end_of_turn>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": true\n    },\n    {\n      \"id\": 108,\n      \"content\": \"\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 109,\n      \"content\": \"\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 110,\n      \"content\": \"\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 111,\n      \"content\": \"\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 112,\n      \"content\": \"\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 113,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 114,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 115,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 116,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 117,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 118,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 119,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 120,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 121,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 122,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 123,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 124,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 125,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 126,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 127,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 128,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 129,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 130,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 131,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 132,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 133,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 134,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 135,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 136,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 137,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 138,\n      \"content\": \"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 139,\n      \"content\": \"▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 140,\n      \"content\": \"▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 141,\n      \"content\": \"▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 142,\n      \"content\": \"▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 143,\n      \"content\": \"▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 144,\n      \"content\": \"▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 145,\n      \"content\": \"▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 146,\n      \"content\": \"▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 147,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 148,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 149,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 150,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 151,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 152,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 153,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 154,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 155,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 156,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 157,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 158,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 159,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 160,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 161,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 162,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 163,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 164,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 165,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 166,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 167,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 168,\n      \"content\": \"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 169,\n      \"content\": \"<table>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 170,\n      \"content\": \"<caption>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 171,\n      \"content\": \"<thead>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 172,\n      \"content\": \"<tbody>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 173,\n      \"content\": \"<tfoot>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 174,\n      \"content\": \"<tr>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 175,\n      \"content\": \"<th>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 176,\n      \"content\": \"<td>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 177,\n      \"content\": \"</table>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 178,\n      \"content\": \"</caption>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 179,\n      \"content\": \"</thead>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 180,\n      \"content\": \"</tbody>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 181,\n      \"content\": \"</tfoot>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 182,\n      \"content\": \"</tr>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 183,\n      \"content\": \"</th>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 184,\n      \"content\": \"</td>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 185,\n      \"content\": \"<h1>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 186,\n      \"content\": \"<h2>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 187,\n      \"content\": \"<h3>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 188,\n      \"content\": \"<h4>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 189,\n      \"content\": \"<h5>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 190,\n      \"content\": \"<h6>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 191,\n      \"content\": \"<blockquote>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 192,\n      \"content\": \"</h1>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 193,\n      \"content\": \"</h2>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 194,\n      \"content\": \"</h3>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 195,\n      \"content\": \"</h4>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 196,\n      \"content\": \"</h5>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 197,\n      \"content\": \"</h6>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 198,\n      \"content\": \"</blockquote>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 199,\n      \"content\": \"<strong>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 200,\n      \"content\": \"<em>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 201,\n      \"content\": \"<b>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 202,\n      \"content\": \"<i>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 203,\n      \"content\": \"<u>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 204,\n      \"content\": \"<s>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 205,\n      \"content\": \"<sub>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 206,\n      \"content\": \"<sup>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 207,\n      \"content\": \"<code>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 208,\n      \"content\": \"</strong>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 209,\n      \"content\": \"</em>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 210,\n      \"content\": \"</b>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 211,\n      \"content\": \"</i>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 212,\n      \"content\": \"</u>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 213,\n      \"content\": \"</s>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 214,\n      \"content\": \"</sub>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 215,\n      \"content\": \"</sup>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    },\n    {\n      \"id\": 216,\n      \"content\": \"</code>\",\n      \"single_word\": false,\n      \"lstrip\": false,\n      \"rstrip\": false,\n      \"normalized\": false,\n      \"special\": false\n    }\n  ],\n  \"normalizer\": {\n    \"type\": \"Replace\",\n    \"pattern\": {\n      \"String\": \" \"\n    },\n    \"content\": \"▁\"\n  },\n  \"pre_tokenizer\": null,\n  \"post_processor\": {\n    \"type\": \"TemplateProcessing\",\n    \"single\": [\n      {\n        \"SpecialToken\": {\n          \"id\": \"<bos>\",\n          \"type_id\": 0\n        }\n      },\n      {\n        \"Sequence\": {\n          \"id\": \"A\",\n          \"type_id\": 0\n        }\n      }\n    ],\n    \"pair\": [\n      {\n        \"SpecialToken\": {\n          \"id\": \"<bos>\",\n          \"type_id\": 0\n        }\n      },\n      {\n        \"Sequence\": {\n          \"id\": \"A\",\n          \"type_id\": 0\n        }\n      },\n      {\n        \"SpecialToken\": {\n          \"id\": \"<bos>\",\n          \"type_id\": 1\n        }\n      },\n      {\n        \"Sequence\": {\n          \"id\": \"B\",\n          \"type_id\": 1\n        }\n      }\n    ],\n    \"special_tokens\": {\n      \"<bos>\": {\n        \"id\": \"<bos>\",\n        \"ids\": [\n          2\n        ],\n        \"tokens\": [\n          \"<bos>\"\n        ]\n      }\n    }\n  },\n  \"decoder\": {\n    \"type\": \"Sequence\",\n    \"decoders\": [\n      {\n        \"type\": \"Replace\",\n        \"pattern\": {\n          \"String\": \"▁\"\n        },\n        \"content\": \" \"\n      },\n      {\n        \"type\": \"ByteFallback\"\n      },\n      {\n        \"type\": \"Fuse\"\n      }\n    ]\n  },\n  \"model\": {\n    \"type\": \"BPE\",\n    \"dropout\": null,\n    \"unk_token\": \"<unk>\",\n    \"continuing_subword_prefix\": null,\n    \"end_of_word_suffix\": null,\n    \"fuse_unk\": true,\n    \"byte_fallback\": true,\n    \"ignore_merges\": false,\n    \"vocab\": {\n      \"<pad>\": 0,\n      \"<eos>\": 1,\n      \"<bos>\": 2,\n      \"<unk>\": 3,\n      \"<mask>\": 4,\n      \"<2mass>\": 5,\n      \"[@BOS@]\": 6,\n      \"<unused0>\": 7,\n      \"<unused1>\": 8,\n      \"<unused2>\": 9,\n      \"<unused3>\": 10,\n      \"<unused4>\": 11,\n      \"<unused5>\": 12,\n      \"<unused6>\": 13,\n      \"<unused7>\": 14,\n      \"<unused8>\": 15,\n      \"<unused9>\": 16,\n      \"<unused10>\": 17,\n      \"<unused11>\": 18,\n      \"<unused12>\": 19,\n      \"<unused13>\": 20,\n      \"<unused14>\": 21,\n      \"<unused15>\": 22,\n      \"<unused16>\": 23,\n      \"<unused17>\": 24,\n      \"<unused18>\": 25,\n      \"<unused19>\": 26,\n      \"<unused20>\": 27,\n      \"<unused21>\": 28,\n      \"<unused22>\": 29,\n      \"<unused23>\": 30,\n      \"<unused24>\": 31,\n      \"<unused25>\": 32,\n      \"<unused26>\": 33,\n      \"<unused27>\": 34,\n      \"<unused28>\": 35,\n      \"<unused29>\": 36,\n      \"<unused30>\": 37,\n      \"<unused31>\": 38,\n      \"<unused32>\": 39,\n      \"<unused33>\": 40,\n      \"<unused34>\": 41,\n      \"<unused35>\": 42,\n      \"<unused36>\": 43,\n      \"<unused37>\": 44,\n      \"<unused38>\": 45,\n      \"<unused39>\": 46,\n      \"<unused40>\": 47,\n      \"<unused41>\": 48,\n      \"<unused42>\": 49,\n      \"<unused43>\": 50,\n      \"<unused44>\": 51,\n      \"<unused45>\": 52,\n      \"<unused46>\": 53,\n      \"<unused47>\":