575 lines
56 KiB
C++
575 lines
56 KiB
C++
//
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018
|
|
//
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
//
|
|
#include "td/utils/unicode.h"
|
|
|
|
#include "td/utils/logging.h"
|
|
|
|
#include <algorithm>
|
|
#include <iterator>
|
|
|
|
namespace td {
|
|
|
|
// list of [(range_begin << 5) + range_type]
|
|
static const uint32 unicode_simple_category_ranges[] = {
|
|
0, 1028, 1056, 1538, 1856, 2081, 2912, 3105, 3936, 5124, 5152, 5441,
|
|
5472, 5699, 5760, 5793, 5824, 5923, 5953, 5984, 6019, 6112, 6145, 6880,
|
|
6913, 7904, 7937, 22592, 22721, 23104, 23553, 23712, 23937, 23968, 24001, 24032,
|
|
28161, 28320, 28353, 28416, 28481, 28608, 28641, 28672, 28865, 28896, 28929, 29024,
|
|
29057, 29088, 29121, 29760, 29793, 32448, 32481, 36928, 37185, 42496, 42529, 43744,
|
|
43809, 43840, 44065, 45312, 47617, 48480, 48641, 48736, 50177, 51552, 52226, 52544,
|
|
52673, 52736, 52769, 55936, 55969, 56000, 56481, 56544, 56769, 56834, 57153, 57248,
|
|
57313, 57344, 57857, 57888, 57921, 58880, 59809, 62656, 63009, 63040, 63490, 63809,
|
|
64864, 65153, 65216, 65345, 65376, 65537, 66240, 66369, 66400, 66689, 66720, 66817,
|
|
66848, 67585, 68384, 70657, 71328, 71361, 71616, 73857, 75584, 75681, 75712, 76289,
|
|
76320, 76545, 76864, 76994, 77312, 77345, 77856, 77985, 78240, 78305, 78368, 78433,
|
|
79136, 79169, 79392, 79425, 79456, 79553, 79680, 79777, 79808, 80321, 80352, 80769,
|
|
80832, 80865, 80960, 81090, 81409, 81472, 81539, 81728, 82081, 82272, 82401, 82464,
|
|
82529, 83232, 83265, 83488, 83521, 83584, 83617, 83680, 83713, 83776, 84769, 84896,
|
|
84929, 84960, 85186, 85504, 85569, 85664, 86177, 86464, 86497, 86592, 86625, 87328,
|
|
87361, 87584, 87617, 87680, 87713, 87872, 87969, 88000, 88577, 88608, 89089, 89152,
|
|
89282, 89600, 89889, 89920, 90273, 90528, 90593, 90656, 90721, 91424, 91457, 91680,
|
|
91713, 91776, 91809, 91968, 92065, 92096, 93057, 93120, 93153, 93248, 93378, 93696,
|
|
93729, 93763, 93952, 94305, 94336, 94369, 94560, 94657, 94752, 94785, 94912, 95009,
|
|
95072, 95105, 95136, 95169, 95232, 95329, 95392, 95489, 95584, 95681, 96064, 96769,
|
|
96800, 97474, 97795, 97888, 98465, 98720, 98753, 98848, 98881, 99616, 99649, 100160,
|
|
100257, 100288, 101121, 101216, 101377, 101440, 101570, 101888, 102147, 102368, 102401, 102432,
|
|
102561, 102816, 102849, 102944, 102977, 103712, 103745, 104064, 104097, 104256, 104353, 104384,
|
|
105409, 105440, 105473, 105536, 105666, 105984, 106017, 106080, 106657, 106912, 106945, 107040,
|
|
107073, 108384, 108449, 108480, 108993, 109024, 109185, 109280, 109315, 109537, 109632, 109762,
|
|
110083, 110368, 110401, 110592, 110753, 111328, 111425, 112192, 112225, 112512, 112545, 112576,
|
|
112641, 112864, 113858, 114176, 114721, 116256, 116289, 116352, 116737, 116960, 117250, 117568,
|
|
118817, 118880, 118913, 118944, 119009, 119072, 119105, 119136, 119201, 119232, 119425, 119552,
|
|
119585, 119808, 119841, 119936, 119969, 120000, 120033, 120064, 120129, 120192, 120225, 120352,
|
|
120385, 120448, 120737, 120768, 120833, 120992, 121025, 121056, 121346, 121664, 121729, 121856,
|
|
122881, 122912, 123906, 124227, 124544, 124929, 125184, 125217, 126368, 127233, 127392, 131073,
|
|
132448, 133089, 133122, 133440, 133633, 133824, 133953, 134080, 134177, 134208, 134305, 134368,
|
|
134593, 134688, 134817, 135232, 135617, 135648, 135682, 136000, 136193, 137408, 137441, 137472,
|
|
137633, 137664, 137729, 139104, 139137, 149792, 149825, 149952, 150017, 150240, 150273, 150304,
|
|
150337, 150464, 150529, 151840, 151873, 152000, 152065, 153120, 153153, 153280, 153345, 153568,
|
|
153601, 153632, 153665, 153792, 153857, 154336, 154369, 156192, 156225, 156352, 156417, 158560,
|
|
159011, 159648, 159745, 160256, 160769, 163520, 163585, 163776, 163873, 183712, 183777, 184324,
|
|
184353, 185184, 185345, 187744, 187843, 187937, 188192, 188417, 188832, 188865, 188992, 189441,
|
|
190016, 190465, 191040, 191489, 191904, 191937, 192032, 192513, 194176, 195297, 195328, 195457,
|
|
195488, 195586, 195904, 196099, 196416, 197122, 197440, 197633, 200448, 200705, 200864, 200929,
|
|
202016, 202049, 202080, 202241, 204480, 204801, 205792, 207042, 207361, 208320, 208385, 208544,
|
|
208897, 210304, 210433, 211264, 211458, 211779, 211808, 212993, 213728, 214017, 215712, 217090,
|
|
217408, 217602, 217920, 218337, 218368, 221345, 222848, 223393, 223616, 223746, 224064, 225377,
|
|
226336, 226753, 226818, 227137, 228544, 229377, 230528, 231426, 231744, 231841, 231938, 232257,
|
|
233408, 233473, 233760, 236833, 236960, 236993, 237120, 237217, 237280, 237569, 243712, 245761,
|
|
254656, 254721, 254912, 254977, 256192, 256257, 256448, 256513, 256768, 256801, 256832, 256865,
|
|
256896, 256929, 256960, 256993, 257984, 258049, 259744, 259777, 260000, 260033, 260064, 260161,
|
|
260256, 260289, 260512, 260609, 260736, 260801, 260992, 261121, 261536, 261697, 261792, 261825,
|
|
262048, 262148, 262496, 263428, 263488, 263652, 263680, 265188, 265216, 265731, 265761, 265792,
|
|
265859, 266048, 266209, 266243, 266560, 266753, 267168, 270401, 270432, 270561, 270592, 270657,
|
|
270976, 271009, 271040, 271137, 271296, 271489, 271520, 271553, 271584, 271617, 271648, 271681,
|
|
271808, 271841, 272192, 272257, 272384, 272545, 272704, 272833, 272864, 272899, 274529, 274595,
|
|
274752, 297987, 299904, 302403, 303104, 323267, 324224, 360449, 361952, 361985, 363488, 363521,
|
|
367776, 367969, 368096, 368193, 368256, 368547, 368576, 368641, 369856, 369889, 369920, 370081,
|
|
370112, 370177, 371968, 372193, 372224, 372737, 373472, 373761, 373984, 374017, 374240, 374273,
|
|
374496, 374529, 374752, 374785, 375008, 375041, 375264, 375297, 375520, 375553, 375776, 378337,
|
|
378368, 393220, 393248, 393377, 393443, 393472, 394275, 394560, 394785, 394944, 395011, 395105,
|
|
395168, 395297, 398048, 398241, 398336, 398369, 401248, 401281, 401408, 401569, 402880, 402977,
|
|
405984, 406083, 406208, 406529, 407392, 409089, 409600, 410627, 410944, 411907, 412160, 412195,
|
|
412672, 413699, 414016, 415267, 415744, 425985, 636608, 638977, 1309376, 1310721, 1348000, 1350145,
|
|
1351616, 1351681, 1360288, 1360385, 1360898, 1361217, 1361280, 1361921, 1363424, 1363937, 1364928, 1364993,
|
|
1367235, 1367552, 1368801, 1369088, 1369153, 1372448, 1372513, 1373664, 1373697, 1373952, 1375969, 1376320,
|
|
1376353, 1376448, 1376481, 1376608, 1376641, 1377376, 1377795, 1377984, 1378305, 1379968, 1380417, 1382016,
|
|
1382914, 1383232, 1384001, 1384192, 1384289, 1384320, 1384353, 1384384, 1384450, 1384769, 1385664, 1385985,
|
|
1386720, 1387521, 1388448, 1388673, 1390176, 1391073, 1391106, 1391424, 1391617, 1391776, 1391809, 1392130,
|
|
1392449, 1392608, 1392641, 1393952, 1394689, 1394784, 1394817, 1395072, 1395202, 1395520, 1395713, 1396448,
|
|
1396545, 1396576, 1396673, 1398272, 1398305, 1398336, 1398433, 1398496, 1398561, 1398720, 1398785, 1398816,
|
|
1398849, 1398880, 1399649, 1399744, 1399809, 1400160, 1400385, 1400480, 1400865, 1401056, 1401121, 1401312,
|
|
1401377, 1401568, 1401857, 1402080, 1402113, 1402336, 1402369, 1403744, 1403777, 1404096, 1404417, 1408096,
|
|
1408514, 1408832, 1409025, 1766528, 1766913, 1767648, 1767777, 1769344, 2039809, 2051520, 2051585, 2054976,
|
|
2056193, 2056416, 2056801, 2056960, 2057121, 2057152, 2057185, 2057504, 2057537, 2057952, 2057985, 2058144,
|
|
2058177, 2058208, 2058241, 2058304, 2058337, 2058400, 2058433, 2061888, 2062945, 2074560, 2075137, 2077184,
|
|
2077249, 2078976, 2080257, 2080640, 2084353, 2084512, 2084545, 2088864, 2089474, 2089792, 2090017, 2090848,
|
|
2091041, 2091872, 2092225, 2095072, 2095169, 2095360, 2095425, 2095616, 2095681, 2095872, 2095937, 2096032,
|
|
2097153, 2097536, 2097569, 2098400, 2098433, 2099040, 2099073, 2099136, 2099169, 2099648, 2099713, 2100160,
|
|
2101249, 2105184, 2105571, 2107008, 2107395, 2109216, 2109763, 2109824, 2117633, 2118560, 2118657, 2120224,
|
|
2120739, 2121600, 2121729, 2122755, 2122880, 2123265, 2123811, 2123841, 2124099, 2124128, 2124289, 2125504,
|
|
2125825, 2126784, 2126849, 2128000, 2128129, 2128384, 2128419, 2128576, 2129921, 2134976, 2135042, 2135360,
|
|
2135553, 2136704, 2136833, 2137984, 2138113, 2139392, 2139649, 2141312, 2146305, 2156256, 2156545, 2157248,
|
|
2157569, 2157824, 2162689, 2162880, 2162945, 2162976, 2163009, 2164416, 2164449, 2164512, 2164609, 2164640,
|
|
2164705, 2165440, 2165507, 2165761, 2166496, 2166563, 2166785, 2167776, 2168035, 2168320, 2169857, 2170464,
|
|
2170497, 2170560, 2170723, 2170881, 2171587, 2171776, 2171905, 2172736, 2174977, 2176768, 2176899, 2176961,
|
|
2177027, 2177536, 2177603, 2179073, 2179104, 2179585, 2179712, 2179745, 2179840, 2179873, 2180736, 2181123,
|
|
2181376, 2182145, 2183075, 2183136, 2183169, 2184099, 2184192, 2185217, 2185472, 2185505, 2186400, 2186595,
|
|
2186752, 2187265, 2188992, 2189313, 2190016, 2190083, 2190337, 2190944, 2191107, 2191361, 2191936, 2192675,
|
|
2192896, 2195457, 2197792, 2199553, 2201184, 2201601, 2203232, 2203459, 2203648, 2214915, 2215904, 2228321,
|
|
2230016, 2230851, 2231490, 2231808, 2232417, 2233856, 2234881, 2235680, 2235906, 2236224, 2236513, 2237664,
|
|
2238146, 2238464, 2238977, 2240096, 2240193, 2240224, 2240609, 2242144, 2242593, 2242720, 2243074, 2243393,
|
|
2243424, 2243457, 2243488, 2243619, 2244256, 2244609, 2245184, 2245217, 2246016, 2248705, 2248928, 2248961,
|
|
2248992, 2249025, 2249152, 2249185, 2249664, 2249697, 2250016, 2250241, 2251744, 2252290, 2252608, 2252961,
|
|
2253216, 2253281, 2253344, 2253409, 2254112, 2254145, 2254368, 2254401, 2254464, 2254497, 2254656, 2254753,
|
|
2254784, 2255361, 2255392, 2255777, 2255936, 2260993, 2262688, 2263265, 2263392, 2263554, 2263872, 2265089,
|
|
2266624, 2267265, 2267328, 2267361, 2267392, 2267650, 2267968, 2273281, 2274784, 2276097, 2276224, 2277377,
|
|
2278912, 2279553, 2279584, 2279938, 2280256, 2281473, 2282848, 2283522, 2283840, 2285569, 2286400, 2287106,
|
|
2287427, 2287488, 2298881, 2300930, 2301251, 2301536, 2301921, 2301952, 2316289, 2318112, 2326529, 2326816,
|
|
2326849, 2328032, 2328577, 2328608, 2329090, 2329411, 2330016, 2330177, 2331136, 2359297, 2388800, 2392067,
|
|
2395616, 2396161, 2402432, 2490369, 2524640, 2654209, 2672864, 2949121, 2967328, 2967553, 2968544, 2968578,
|
|
2968896, 2972161, 2973120, 2973697, 2975232, 2975745, 2975872, 2976258, 2976576, 2976611, 2976832, 2976865,
|
|
2977536, 2977697, 2978304, 3006465, 3008672, 3009025, 3009056, 3011169, 3011584, 3013633, 3013664, 3014657,
|
|
3210656, 3211265, 3235424, 3538945, 3539008, 3637249, 3640672, 3640833, 3641248, 3641345, 3641632, 3641857,
|
|
3642176, 3828739, 3829312, 3833857, 3836576, 3836609, 3838880, 3838913, 3838976, 3839041, 3839072, 3839137,
|
|
3839200, 3839265, 3839392, 3839425, 3839808, 3839841, 3839872, 3839905, 3840128, 3840161, 3842240, 3842273,
|
|
3842400, 3842465, 3842720, 3842753, 3842976, 3843009, 3843904, 3843937, 3844064, 3844097, 3844256, 3844289,
|
|
3844320, 3844417, 3844640, 3844673, 3855552, 3855617, 3856416, 3856449, 3857248, 3857281, 3858272, 3858305,
|
|
3859104, 3859137, 3860128, 3860161, 3860960, 3860993, 3861984, 3862017, 3862816, 3862849, 3863840, 3863873,
|
|
3864672, 3864705, 3864960, 3865026, 3866624, 3997697, 4004000, 4004067, 4004352, 4005889, 4008064, 4008450,
|
|
4008768, 4046849, 4046976, 4047009, 4047872, 4047905, 4047968, 4048001, 4048032, 4048097, 4048128, 4048161,
|
|
4048480, 4048513, 4048640, 4048673, 4048704, 4048737, 4048768, 4048961, 4048992, 4049121, 4049152, 4049185,
|
|
4049216, 4049249, 4049280, 4049313, 4049408, 4049441, 4049504, 4049537, 4049568, 4049633, 4049664, 4049697,
|
|
4049728, 4049761, 4049792, 4049825, 4049856, 4049889, 4049920, 4049953, 4050016, 4050049, 4050080, 4050145,
|
|
4050272, 4050305, 4050528, 4050561, 4050688, 4050721, 4050848, 4050881, 4050912, 4050945, 4051264, 4051297,
|
|
4051840, 4052001, 4052096, 4052129, 4052288, 4052321, 4052864, 4071427, 4071840, 4194305, 5561056, 5562369,
|
|
5695136, 5695489, 5702592, 5702657, 5887040, 6225921, 6243264, 4294967295};
|
|
|
|
static constexpr uint32 TABLE_SIZE = 1280;
|
|
|
|
static int16 prepare_search_character_table[TABLE_SIZE] = {
|
|
0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
|
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
|
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 48, 49, 50, 51, 52, 53, 54, 55, 56,
|
|
57, 32, 32, 32, 32, 32, 32, 32, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107,
|
|
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 32, 32, 32, 32,
|
|
32, 32, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113,
|
|
114, 115, 116, 117, 118, 119, 120, 121, 122, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
|
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
|
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 170,
|
|
32, 32, 0, 32, 32, 32, 32, 178, 179, 32, 956, 32, 0, 32, 185, 186, 32, 188, 189,
|
|
190, 32, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240,
|
|
241, 242, 243, 244, 245, 246, 32, 248, 249, 250, 251, 252, 253, 254, 223, 224, 225, 226, 227,
|
|
228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,
|
|
32, 248, 249, 250, 251, 252, 253, 254, 255, 257, 257, 259, 259, 261, 261, 263, 263, 265, 265,
|
|
267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277, 279, 279, 281, 281, 283, 283, 285,
|
|
285, 287, 287, 289, 289, 291, 291, 293, 293, 295, 295, 297, 297, 299, 299, 301, 301, 303, 303,
|
|
105, 305, 307, 307, 309, 309, 311, 311, 312, 314, 314, 316, 316, 318, 318, 320, 320, 322, 322,
|
|
324, 324, 326, 326, 328, 328, 329, 331, 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341,
|
|
343, 343, 345, 345, 347, 347, 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361,
|
|
361, 363, 363, 365, 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 255, 378, 378, 380,
|
|
380, 382, 382, 115, 384, 595, 387, 387, 389, 389, 596, 392, 392, 598, 599, 396, 396, 397, 477,
|
|
601, 603, 402, 402, 608, 611, 405, 617, 616, 409, 409, 410, 411, 623, 626, 414, 629, 417, 417,
|
|
419, 419, 421, 421, 640, 424, 424, 643, 426, 427, 429, 429, 648, 432, 432, 650, 651, 436, 436,
|
|
438, 438, 658, 441, 441, 442, 443, 445, 445, 446, 447, 448, 449, 450, 451, 454, 454, 454, 457,
|
|
457, 457, 460, 460, 460, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470, 472, 472, 474, 474,
|
|
476, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, 487, 487, 489, 489, 491, 491, 493, 493,
|
|
495, 495, 496, 499, 499, 499, 501, 501, 405, 447, 505, 505, 507, 507, 509, 509, 511, 511, 513,
|
|
513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, 525, 525, 527, 527, 529, 529, 531, 531,
|
|
533, 533, 535, 535, 537, 537, 539, 539, 541, 541, 543, 543, 414, 545, 547, 547, 549, 549, 551,
|
|
551, 553, 553, 555, 555, 557, 557, 559, 559, 561, 561, 563, 563, 564, 565, 566, 567, 568, 569,
|
|
11365, 572, 572, 410, 11366, 575, 576, 578, 578, 384, 649, 652, 583, 583, 585, 585, 587, 587, 589,
|
|
589, 591, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607,
|
|
608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626,
|
|
627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645,
|
|
646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664,
|
|
665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683,
|
|
684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 0, 701, 0,
|
|
703, 704, 705, 32, 32, 32, 32, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721,
|
|
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 736, 737, 738, 739, 740,
|
|
32, 32, 32, 32, 32, 32, 32, 748, 32, 750, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
|
32, 32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 881, 881, 883, 883, 884, 32, 887, 887, 888, 889, 890, 891, 892,
|
|
893, 32, 1011, 896, 897, 898, 899, 32, 32, 940, 32, 941, 942, 943, 907, 972, 909, 973, 974,
|
|
912, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 930,
|
|
963, 964, 965, 966, 967, 968, 969, 970, 971, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949,
|
|
950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 963, 963, 964, 965, 966, 967, 968,
|
|
969, 970, 971, 972, 973, 974, 983, 946, 952, 965, 965, 965, 966, 960, 983, 985, 985, 987, 987,
|
|
989, 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005, 1005, 1007,
|
|
1007, 954, 961, 963, 1011, 952, 949, 32, 1016, 1016, 1010, 1019, 1019, 1020, 891, 892, 893, 1104, 1105,
|
|
1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1072, 1073, 1074, 1075, 1076,
|
|
1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095,
|
|
1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082,
|
|
1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101,
|
|
1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121,
|
|
1121, 1123, 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137, 1137, 1139, 1139,
|
|
1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151, 1151, 1153, 1153, 32, 0, 0, 0, 0,
|
|
0, 0, 0, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173, 1173, 1175, 1175, 1177, 1177,
|
|
1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187, 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197,
|
|
1197, 1199, 1199, 1201, 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215, 1215,
|
|
1231, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230, 1230, 1231, 1233, 1233, 1235,
|
|
1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245, 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253,
|
|
1255, 1255, 1257, 1257, 1259, 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273,
|
|
1273, 1275, 1275, 1277, 1277, 1279, 1279};
|
|
|
|
static const int32 prepare_search_character_ranges[] = {
|
|
1280, 2097153, 1328, 1328, 1329, -1378, 1367, -1368, 1370, 32, 1376, -1377,
|
|
1417, 32, 1419, -1420, 1421, 32, 1424, 1424, 1425, 0, 1470, 32,
|
|
1471, 0, 1472, 32, 1473, 0, 1475, 32, 1476, 0, 1478, 32,
|
|
1479, 0, 1480, -1481, 1523, 32, 1525, -1526, 1536, 0, 1542, 32,
|
|
1552, 0, 1563, 32, 1564, 0, 1565, 1565, 1566, 32, 1568, -1569,
|
|
1611, 0, 1632, -1633, 1642, 32, 1646, -1647, 1648, 0, 1649, -1650,
|
|
1748, 32, 1749, 1749, 1750, 0, 1758, 32, 1759, 0, 1765, -1766,
|
|
1767, 0, 1769, 32, 1770, 0, 1774, -1775, 1789, 32, 1791, 1791,
|
|
1792, 32, 1806, 1806, 1807, 0, 1808, 1808, 1809, 0, 1810, -1811,
|
|
1840, 0, 1867, -1868, 1958, 0, 1969, -1970, 2027, 0, 2036, -2037,
|
|
2038, 32, 2042, -2043, 2070, 0, 2074, 2074, 2075, 0, 2084, 2084,
|
|
2085, 0, 2088, 2088, 2089, 0, 2094, -2095, 2096, 32, 2111, -2112,
|
|
2137, 0, 2140, -2141, 2142, 32, 2143, -2144, 2260, 0, 2308, -2309,
|
|
2362, 0, 2365, 2365, 2366, 0, 2384, 2384, 2385, 0, 2392, -2393,
|
|
2402, 0, 2404, 32, 2406, -2407, 2416, 32, 2417, -2418, 2433, 0,
|
|
2436, -2437, 2492, 0, 2493, 2493, 2494, 0, 2501, -2502, 2503, 0,
|
|
2505, -2506, 2507, 0, 2510, -2511, 2519, 0, 2520, -2521, 2530, 0,
|
|
2532, -2533, 2546, 32, 2548, -2549, 2554, 32, 2556, -2557, 2561, 0,
|
|
2564, -2565, 2620, 0, 2621, 2621, 2622, 0, 2627, -2628, 2631, 0,
|
|
2633, -2634, 2635, 0, 2638, -2639, 2641, 0, 2642, -2643, 2672, 0,
|
|
2674, -2675, 2677, 0, 2678, -2679, 2689, 0, 2692, -2693, 2748, 0,
|
|
2749, 2749, 2750, 0, 2758, 2758, 2759, 0, 2762, 2762, 2763, 0,
|
|
2766, -2767, 2786, 0, 2788, -2789, 2800, 32, 2802, -2803, 2817, 0,
|
|
2820, -2821, 2876, 0, 2877, 2877, 2878, 0, 2885, -2886, 2887, 0,
|
|
2889, -2890, 2891, 0, 2894, -2895, 2902, 0, 2904, -2905, 2914, 0,
|
|
2916, -2917, 2928, 32, 2929, -2930, 2946, 0, 2947, -2948, 3006, 0,
|
|
3011, -3012, 3014, 0, 3017, 3017, 3018, 0, 3022, -3023, 3031, 0,
|
|
3032, -3033, 3059, 32, 3067, -3068, 3072, 0, 3076, -3077, 3134, 0,
|
|
3141, 3141, 3142, 0, 3145, 3145, 3146, 0, 3150, -3151, 3157, 0,
|
|
3159, -3160, 3170, 0, 3172, -3173, 3199, 32, 3200, 3200, 3201, 0,
|
|
3204, -3205, 3260, 0, 3261, 3261, 3262, 0, 3269, 3269, 3270, 0,
|
|
3273, 3273, 3274, 0, 3278, -3279, 3285, 0, 3287, -3288, 3298, 0,
|
|
3300, -3301, 3329, 0, 3332, -3333, 3390, 0, 3397, 3397, 3398, 0,
|
|
3401, 3401, 3402, 0, 3406, 3406, 3407, 32, 3408, -3409, 3415, 0,
|
|
3416, -3417, 3426, 0, 3428, -3429, 3449, 32, 3450, -3451, 3458, 0,
|
|
3460, -3461, 3530, 0, 3531, -3532, 3535, 0, 3541, 3541, 3542, 0,
|
|
3543, 3543, 3544, 0, 3552, -3553, 3570, 0, 3572, 32, 3573, -3574,
|
|
3633, 0, 3634, -3635, 3636, 0, 3643, -3644, 3647, 32, 3648, -3649,
|
|
3655, 0, 3663, 32, 3664, -3665, 3674, 32, 3676, -3677, 3761, 0,
|
|
3762, -3763, 3764, 0, 3770, 3770, 3771, 0, 3773, -3774, 3784, 0,
|
|
3790, -3791, 3841, 32, 3864, 0, 3866, 32, 3872, -3873, 3892, 32,
|
|
3893, 0, 3894, 32, 3895, 0, 3896, 32, 3897, 0, 3898, 32,
|
|
3902, 0, 3904, -3905, 3953, 0, 3973, 32, 3974, 0, 3976, -3977,
|
|
3981, 0, 3992, 3992, 3993, 0, 4029, 4029, 4030, 32, 4038, 0,
|
|
4039, 32, 4045, 4045, 4046, 32, 4059, -4060, 4139, 0, 4159, -4160,
|
|
4170, 32, 4176, -4177, 4182, 0, 4186, -4187, 4190, 0, 4193, 4193,
|
|
4194, 0, 4197, -4198, 4199, 0, 4206, -4207, 4209, 0, 4213, -4214,
|
|
4226, 0, 4238, 4238, 4239, 0, 4240, -4241, 4250, 0, 4254, 32,
|
|
4256, -11521, 4294, 4294, 4295, 11559, 4296, -4297, 4301, 11565, 4302, -4303,
|
|
4347, 32, 4348, -4349, 4957, 0, 4960, 32, 4969, -4970, 5008, 32,
|
|
5018, -5019, 5112, -5105, 5118, -5119, 5120, 32, 5121, -5122, 5741, 32,
|
|
5743, -5744, 5760, 32, 5761, -5762, 5787, 32, 5789, -5790, 5867, 32,
|
|
5870, -5871, 5906, 0, 5909, -5910, 5938, 0, 5941, 32, 5943, -5944,
|
|
5970, 0, 5972, -5973, 6002, 0, 6004, -6005, 6068, 0, 6100, 32,
|
|
6103, 6103, 6104, 32, 6108, 6108, 6109, 0, 6110, -6111, 6144, 32,
|
|
6155, 0, 6159, -6160, 6277, 0, 6279, -6280, 6313, 0, 6314, -6315,
|
|
6432, 0, 6444, -6445, 6448, 0, 6460, -6461, 6464, 32, 6465, -6466,
|
|
6468, 32, 6470, -6471, 6622, 32, 6656, -6657, 6679, 0, 6684, -6685,
|
|
6686, 32, 6688, -6689, 6741, 0, 6751, 6751, 6752, 0, 6781, -6782,
|
|
6783, 0, 6784, -6785, 6816, 32, 6823, 6823, 6824, 32, 6830, -6831,
|
|
6832, 0, 6847, -6848, 6912, 0, 6917, -6918, 6964, 0, 6981, -6982,
|
|
7002, 32, 7019, 0, 7028, 32, 7037, -7038, 7040, 0, 7043, -7044,
|
|
7073, 0, 7086, -7087, 7142, 0, 7156, -7157, 7164, 32, 7168, -7169,
|
|
7204, 0, 7224, -7225, 7227, 32, 7232, -7233, 7294, 32, 7296, 1074,
|
|
7297, 1076, 7298, 1086, 7299, -1090, 7301, 1090, 7302, 1098, 7303, 1123,
|
|
7304, 42571, 7305, -7306, 7360, 32, 7368, -7369, 7376, 0, 7379, 32,
|
|
7380, 0, 7401, -7402, 7405, 0, 7406, -7407, 7410, 0, 7413, -7414,
|
|
7416, 0, 7418, -7419, 7468, 97, 7469, 230, 7470, 98, 7471, 7471,
|
|
7472, -101, 7474, 477, 7475, -104, 7483, 7483, 7484, 111, 7485, 547,
|
|
7486, 112, 7487, 114, 7488, -117, 7490, 119, 7491, -7492, 7616, 0,
|
|
7670, -7671, 7675, 0, 7680, 2097153, 7830, -7831, 7835, 7777, 7836, -7837,
|
|
7838, 223, 7839, 2097153, 7936, -7937, 7944, -7937, 7952, -7953, 7960, -7953,
|
|
7966, -7967, 7976, -7969, 7984, -7985, 7992, -7985, 8000, -8001, 8008, -8001,
|
|
8014, -8015, 8025, 8017, 8026, 8026, 8027, 8019, 8028, 8028, 8029, 8021,
|
|
8030, 8030, 8031, 8023, 8032, -8033, 8040, -8033, 8048, -8049, 8072, -8065,
|
|
8080, -8081, 8088, -8081, 8096, -8097, 8104, -8097, 8112, -8113, 8120, -8113,
|
|
8122, -8049, 8124, 8115, 8125, 32, 8126, 953, 8127, 32, 8130, -8131,
|
|
8136, -8051, 8140, 8131, 8141, 32, 8144, -8145, 8152, -8145, 8154, -8055,
|
|
8156, 8156, 8157, 32, 8160, -8161, 8168, -8161, 8170, -8059, 8172, 8165,
|
|
8173, 32, 8176, -8177, 8184, -8057, 8186, -8061, 8188, 8179, 8189, 32,
|
|
8191, 8191, 8192, 32, 8203, 0, 8208, 32, 8234, 0, 8239, 32,
|
|
8288, 0, 8293, 8293, 8294, 0, 8304, -8305, 8314, 32, 8319, -8320,
|
|
8330, 32, 8335, -8336, 8352, 32, 8383, -8384, 8400, 0, 8433, -8434,
|
|
8448, 32, 8450, 99, 8452, 32, 8455, 603, 8456, 32, 8457, 102,
|
|
8458, 8458, 8459, 104, 8462, -8463, 8464, 105, 8466, 108, 8467, 8467,
|
|
8468, 32, 8469, 110, 8470, 32, 8473, -113, 8476, 114, 8478, 32,
|
|
8484, 122, 8485, 32, 8486, 969, 8487, 32, 8488, 122, 8489, 32,
|
|
8490, 107, 8491, 229, 8492, -99, 8494, 32, 8495, 8495, 8496, -102,
|
|
8498, 8526, 8499, 109, 8500, -8501, 8506, 32, 8508, -8509, 8510, 947,
|
|
8511, 960, 8512, 32, 8517, 100, 8518, -8519, 8522, 32, 8526, 8526,
|
|
8527, 32, 8528, -8529, 8544, -8561, 8560, -8561, 8579, 8580, 8581, -8582,
|
|
8586, 32, 8588, -8589, 8592, 32, 9215, 9215, 9216, 32, 9255, -9256,
|
|
9280, 32, 9291, -9292, 9372, 32, 9398, -9425, 9424, -9425, 9472, 32,
|
|
10102, -10103, 10132, 32, 11124, -11125, 11126, 32, 11158, -11159, 11160, 32,
|
|
11194, -11195, 11197, 32, 11209, 11209, 11210, 32, 11218, -11219, 11244, 32,
|
|
11248, -11249, 11264, -11313, 11311, -11312, 11360, 11361, 11362, 619, 11363, 7549,
|
|
11364, 637, 11365, -11366, 11367, 11368, 11369, 11370, 11371, 11372, 11373, 593,
|
|
11374, 625, 11375, 592, 11376, 594, 11377, 2097153, 11380, 11380, 11381, 11382,
|
|
11383, -11384, 11389, 118, 11390, -576, 11392, 2097153, 11492, 11492, 11493, 32,
|
|
11499, 11500, 11501, 11502, 11503, 0, 11506, 11507, 11508, -11509, 11513, 32,
|
|
11517, 11517, 11518, 32, 11520, -11521, 11632, 32, 11633, -11634, 11647, 0,
|
|
11648, -11649, 11744, 0, 11776, 32, 11823, 11823, 11824, 32, 11845, -11846,
|
|
11904, 32, 11930, 11930, 11931, 32, 11935, 11935, 11936, 32, 12019, -12020,
|
|
12272, 32, 12284, -12285, 12288, 32, 12293, -12294, 12296, 32, 12321, -12322,
|
|
12330, 0, 12336, 32, 12337, -12338, 12342, 32, 12344, -12345, 12349, 32,
|
|
12352, -12353, 12441, 0, 12443, 32, 12445, -12446, 12448, 32, 12449, -12450,
|
|
12539, 32, 12540, 0, 12541, -12542, 12688, 32, 12690, -12691, 12736, 32,
|
|
12772, -12773, 12800, 32, 12831, -12832, 12842, 32, 12868, -12869, 12880, 32,
|
|
12881, -12882, 12910, 32, 12928, -12929, 12992, 32, 13008, -13009, 13056, 32,
|
|
13312, -13313, 19904, 32, 19968, -19969, 42128, 32, 42183, -42184, 42238, 32,
|
|
42240, -42241, 42509, 32, 42512, -42513, 42560, 2097153, 42606, 42606, 42607, 0,
|
|
42611, 32, 42612, 0, 42622, 32, 42623, 2097153, 42652, -42653, 42654, 0,
|
|
42656, -42657, 42736, 0, 42738, 32, 42744, -42745, 42752, 32, 42775, -42776,
|
|
42784, 32, 42786, 2097153, 42800, -42801, 42802, 2097153, 42864, -42865, 42873, 42874,
|
|
42875, 42876, 42877, 7545, 42878, 2097153, 42888, 42888, 42889, 32, 42891, 42892,
|
|
42893, 613, 42894, -42895, 42896, 2097153, 42900, -42901, 42902, 2097153, 42922, 614,
|
|
42923, 604, 42924, 609, 42925, 620, 42926, 618, 42927, 42927, 42928, 670,
|
|
42929, 647, 42930, 669, 42931, 43859, 42932, 2097153, 42936, -42937, 43000, 295,
|
|
43001, -43002, 43010, 0, 43011, -43012, 43014, 0, 43015, -43016, 43019, 0,
|
|
43020, -43021, 43043, 0, 43048, 32, 43052, -43053, 43062, 32, 43066, -43067,
|
|
43124, 32, 43128, -43129, 43136, 0, 43138, -43139, 43188, 0, 43206, -43207,
|
|
43214, 32, 43216, -43217, 43232, 0, 43250, -43251, 43256, 32, 43259, 43259,
|
|
43260, 32, 43261, -43262, 43302, 0, 43310, 32, 43312, -43313, 43335, 0,
|
|
43348, -43349, 43359, 32, 43360, -43361, 43392, 0, 43396, -43397, 43443, 0,
|
|
43457, 32, 43470, -43471, 43486, 32, 43488, -43489, 43493, 0, 43494, -43495,
|
|
43561, 0, 43575, -43576, 43587, 0, 43588, -43589, 43596, 0, 43598, -43599,
|
|
43612, 32, 43616, -43617, 43639, 32, 43642, 43642, 43643, 0, 43646, -43647,
|
|
43696, 0, 43697, 43697, 43698, 0, 43701, -43702, 43703, 0, 43705, -43706,
|
|
43710, 0, 43712, 43712, 43713, 0, 43714, -43715, 43742, 32, 43744, -43745,
|
|
43755, 0, 43760, 32, 43762, -43763, 43765, 0, 43767, -43768, 43867, 32,
|
|
43868, -43869, 43888, -5025, 43968, -43969, 44003, 0, 44011, 32, 44012, 0,
|
|
44014, -44015, 55296, 0, 57344, -57345, 64286, 0, 64287, -64288, 64297, 32,
|
|
64298, -64299, 64434, 32, 64450, -64451, 64830, 32, 64832, -64833, 64976, 32,
|
|
65008, -65009, 65020, 32, 65022, -65023, 65024, 0, 65040, 32, 65050, -65051,
|
|
65056, 0, 65072, 32, 65107, 65107, 65108, 32, 65127, 65127, 65128, 32,
|
|
65132, -65133, 65279, 0, 65280, 65280, 65281, 32, 65296, -65297, 65306, 32,
|
|
65313, -65346, 65339, 32, 65345, -65346, 65371, 32, 65382, -65383, 65504, 32,
|
|
65511, 65511, 65512, 32, 65519, -65520, 65529, 0, 65532, 32, 65536, -65537,
|
|
65792, 32, 65795, -65796, 65847, 32, 65856, -65857, 65913, 32, 65930, -65931,
|
|
65932, 32, 65935, 65935, 65936, 32, 65948, -65949, 65952, 32, 65953, -65954,
|
|
66000, 32, 66045, 0, 66046, -66047, 66272, 0, 66273, -66274, 66422, 0,
|
|
66427, -66428, 66463, 32, 66464, -66465, 66512, 32, 66513, -66514, 66560, -66601,
|
|
66600, -66601, 66736, -66777, 66772, -66773, 66927, 32, 66928, -66929, 67671, 32,
|
|
67672, -67673, 67703, 32, 67705, -67706, 67871, 32, 67872, -67873, 67903, 32,
|
|
67904, -67905, 68097, 0, 68100, 68100, 68101, 0, 68103, -68104, 68108, 0,
|
|
68112, -68113, 68152, 0, 68155, -68156, 68159, 0, 68160, -68161, 68176, 32,
|
|
68185, -68186, 68223, 32, 68224, -68225, 68296, 32, 68297, -68298, 68325, 0,
|
|
68327, -68328, 68336, 32, 68343, -68344, 68409, 32, 68416, -68417, 68505, 32,
|
|
68509, -68510, 68736, -68801, 68787, -68788, 69632, 0, 69635, -69636, 69688, 0,
|
|
69703, 32, 69710, -69711, 69759, 0, 69763, -69764, 69808, 0, 69819, 32,
|
|
69821, 0, 69822, 32, 69826, -69827, 69888, 0, 69891, -69892, 69927, 0,
|
|
69941, -69942, 69952, 32, 69956, -69957, 70003, 0, 70004, 32, 70006, -70007,
|
|
70016, 0, 70019, -70020, 70067, 0, 70081, -70082, 70085, 32, 70090, 0,
|
|
70093, 32, 70094, -70095, 70107, 32, 70108, 70108, 70109, 32, 70112, -70113,
|
|
70188, 0, 70200, 32, 70206, 0, 70207, -70208, 70313, 32, 70314, -70315,
|
|
70367, 0, 70379, -70380, 70400, 0, 70404, -70405, 70460, 0, 70461, 70461,
|
|
70462, 0, 70469, -70470, 70471, 0, 70473, -70474, 70475, 0, 70478, -70479,
|
|
70487, 0, 70488, -70489, 70498, 0, 70500, -70501, 70502, 0, 70509, -70510,
|
|
70512, 0, 70517, -70518, 70709, 0, 70727, -70728, 70731, 32, 70736, -70737,
|
|
70747, 32, 70748, 70748, 70749, 32, 70750, -70751, 70832, 0, 70852, -70853,
|
|
70854, 32, 70855, -70856, 71087, 0, 71094, -71095, 71096, 0, 71105, 32,
|
|
71128, -71129, 71132, 0, 71134, -71135, 71216, 0, 71233, 32, 71236, -71237,
|
|
71264, 32, 71277, -71278, 71339, 0, 71352, -71353, 71453, 0, 71468, -71469,
|
|
71484, 32, 71488, -71489, 71840, -71873, 71872, -71873, 72751, 0, 72759, 72759,
|
|
72760, 0, 72768, 72768, 72769, 32, 72774, -72775, 72816, 32, 72818, -72819,
|
|
72850, 0, 72872, 72872, 72873, 0, 72887, -72888, 74864, 32, 74869, -74870,
|
|
92782, 32, 92784, -92785, 92912, 0, 92917, 32, 92918, -92919, 92976, 0,
|
|
92983, 32, 92992, -92993, 92996, 32, 92998, -92999, 94033, 0, 94079, -94080,
|
|
94095, 0, 94099, -94100, 113820, 32, 113821, 0, 113823, 32, 113824, 0,
|
|
113828, -113829, 118784, 32, 119030, -119031, 119040, 32, 119079, -119080, 119081, 32,
|
|
119141, 0, 119146, 32, 119149, 0, 119171, 32, 119173, 0, 119180, 32,
|
|
119210, 0, 119214, 32, 119273, -119274, 119296, 32, 119362, 0, 119365, 32,
|
|
119366, -119367, 119552, 32, 119639, -119640, 119808, -98, 119834, -119835, 119860, -98,
|
|
119886, -119887, 119912, -98, 119938, -119939, 119964, 97, 119965, 119965, 119966, -100,
|
|
119968, -119969, 119970, 103, 119971, -119972, 119973, -107, 119975, -119976, 119977, -111,
|
|
119981, 119981, 119982, -116, 119990, -119991, 120016, -98, 120042, -120043, 120068, -98,
|
|
120070, 120070, 120071, -101, 120075, -120076, 120077, -107, 120085, 120085, 120086, -116,
|
|
120093, -120094, 120120, -98, 120122, 120122, 120123, -101, 120127, 120127, 120128, -106,
|
|
120133, 120133, 120134, 111, 120135, -120136, 120138, -116, 120145, -120146, 120172, -98,
|
|
120198, -120199, 120224, -98, 120250, -120251, 120276, -98, 120302, -120303, 120328, -98,
|
|
120354, -120355, 120380, -98, 120406, -120407, 120432, -98, 120458, -120459, 120488, -946,
|
|
120505, 952, 120506, -964, 120513, 32, 120514, -120515, 120531, 963, 120532, -120533,
|
|
120539, 32, 120540, -120541, 120546, -946, 120563, 952, 120564, -964, 120571, 32,
|
|
120572, -120573, 120589, 963, 120590, -120591, 120597, 32, 120598, -120599, 120604, -946,
|
|
120621, 952, 120622, -964, 120629, 32, 120630, -120631, 120647, 963, 120648, -120649,
|
|
120655, 32, 120656, -120657, 120662, -946, 120679, 952, 120680, -964, 120687, 32,
|
|
120688, -120689, 120705, 963, 120706, -120707, 120713, 32, 120714, -120715, 120720, -946,
|
|
120737, 952, 120738, -964, 120745, 32, 120746, -120747, 120763, 963, 120764, -120765,
|
|
120771, 32, 120772, -120773, 120778, 989, 120779, -120780, 120832, 32, 121344, 0,
|
|
121399, 32, 121403, 0, 121453, 32, 121461, 0, 121462, 32, 121476, 0,
|
|
121477, 32, 121484, -121485, 121499, 0, 121504, 121504, 121505, 0, 121520, -121521,
|
|
122880, 0, 122887, 122887, 122888, 0, 122905, -122906, 122907, 0, 122914, 122914,
|
|
122915, 0, 122917, 122917, 122918, 0, 122923, -122924, 125136, 0, 125143, -125144,
|
|
125184, -125219, 125218, -125219, 125252, 0, 125259, -125260, 125278, 32, 125280, -125281,
|
|
126704, 32, 126706, -126707, 126976, 32, 127020, -127021, 127024, 32, 127124, -127125,
|
|
127136, 32, 127151, -127152, 127153, 32, 127168, 127168, 127169, 32, 127184, 127184,
|
|
127185, 32, 127222, -127223, 127248, 32, 127275, 99, 127276, 114, 127277, 32,
|
|
127279, 127279, 127280, -98, 127306, 32, 127340, -127341, 127344, 32, 127405, -127406,
|
|
127462, 32, 127490, -127491, 127552, 32, 127561, -127562, 127744, 32, 128723, -128724,
|
|
128736, 32, 128749, -128750, 128752, 32, 128759, -128760, 128768, 32, 128884, -128885,
|
|
128896, 32, 128981, -128982, 129024, 32, 129036, -129037, 129040, 32, 129096, -129097,
|
|
129104, 32, 129114, -129115, 129120, 32, 129160, -129161, 129168, 32, 129198, -129199,
|
|
129296, 32, 129311, 129311, 129312, 32, 129320, -129321, 129328, 32, 129329, -129330,
|
|
129331, 32, 129343, 129343, 129344, 32, 129356, -129357, 129360, 32, 129375, -129376,
|
|
129408, 32, 129426, -129427, 129472, 32, 129473, -129474, 131070, 32, 131072, -131073,
|
|
196606, 32, 196608, -196609, 262142, 32, 262144, -262145, 327678, 32, 327680, -327681,
|
|
393214, 32, 393216, -393217, 458750, 32, 458752, -458753, 524286, 32, 524288, -524289,
|
|
589822, 32, 589824, -589825, 655358, 32, 655360, -655361, 720894, 32, 720896, -720897,
|
|
786430, 32, 786432, -786433, 851966, 32, 851968, -851969, 917502, 32, 917504, 917504,
|
|
917505, 0, 917506, -917507, 917536, 0, 917632, -917633, 917760, 0, 918000, -918001,
|
|
983038, 32, 983040, -983041, 1048574, 32, 1048576, -1048577, 1114110, 32, 2147483647, 0};
|
|
|
|
static int16 to_lower_table[TABLE_SIZE] = {
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
|
|
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
|
|
38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56,
|
|
57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107,
|
|
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 91, 92, 93, 94,
|
|
95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113,
|
|
114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
|
|
133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151,
|
|
152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,
|
|
171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
|
|
190, 191, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240,
|
|
241, 242, 243, 244, 245, 246, 215, 248, 249, 250, 251, 252, 253, 254, 223, 224, 225, 226, 227,
|
|
228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,
|
|
247, 248, 249, 250, 251, 252, 253, 254, 255, 257, 257, 259, 259, 261, 261, 263, 263, 265, 265,
|
|
267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277, 279, 279, 281, 281, 283, 283, 285,
|
|
285, 287, 287, 289, 289, 291, 291, 293, 293, 295, 295, 297, 297, 299, 299, 301, 301, 303, 303,
|
|
105, 305, 307, 307, 309, 309, 311, 311, 312, 314, 314, 316, 316, 318, 318, 320, 320, 322, 322,
|
|
324, 324, 326, 326, 328, 328, 329, 331, 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341,
|
|
343, 343, 345, 345, 347, 347, 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361,
|
|
361, 363, 363, 365, 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 255, 378, 378, 380,
|
|
380, 382, 382, 383, 384, 595, 387, 387, 389, 389, 596, 392, 392, 598, 599, 396, 396, 397, 477,
|
|
601, 603, 402, 402, 608, 611, 405, 617, 616, 409, 409, 410, 411, 623, 626, 414, 629, 417, 417,
|
|
419, 419, 421, 421, 640, 424, 424, 643, 426, 427, 429, 429, 648, 432, 432, 650, 651, 436, 436,
|
|
438, 438, 658, 441, 441, 442, 443, 445, 445, 446, 447, 448, 449, 450, 451, 454, 454, 454, 457,
|
|
457, 457, 460, 460, 460, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470, 472, 472, 474, 474,
|
|
476, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, 487, 487, 489, 489, 491, 491, 493, 493,
|
|
495, 495, 496, 499, 499, 499, 501, 501, 405, 447, 505, 505, 507, 507, 509, 509, 511, 511, 513,
|
|
513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, 525, 525, 527, 527, 529, 529, 531, 531,
|
|
533, 533, 535, 535, 537, 537, 539, 539, 541, 541, 543, 543, 414, 545, 547, 547, 549, 549, 551,
|
|
551, 553, 553, 555, 555, 557, 557, 559, 559, 561, 561, 563, 563, 564, 565, 566, 567, 568, 569,
|
|
11365, 572, 572, 410, 11366, 575, 576, 578, 578, 384, 649, 652, 583, 583, 585, 585, 587, 587, 589,
|
|
589, 591, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607,
|
|
608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626,
|
|
627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645,
|
|
646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664,
|
|
665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683,
|
|
684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702,
|
|
703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721,
|
|
722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740,
|
|
741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759,
|
|
760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778,
|
|
779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797,
|
|
798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816,
|
|
817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835,
|
|
836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854,
|
|
855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873,
|
|
874, 875, 876, 877, 878, 879, 881, 881, 883, 883, 884, 885, 887, 887, 888, 889, 890, 891, 892,
|
|
893, 894, 1011, 896, 897, 898, 899, 900, 901, 940, 903, 941, 942, 943, 907, 972, 909, 973, 974,
|
|
912, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 930,
|
|
963, 964, 965, 966, 967, 968, 969, 970, 971, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949,
|
|
950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968,
|
|
969, 970, 971, 972, 973, 974, 983, 976, 977, 978, 979, 980, 981, 982, 983, 985, 985, 987, 987,
|
|
989, 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005, 1005, 1007,
|
|
1007, 1008, 1009, 1010, 1011, 952, 1013, 1014, 1016, 1016, 1010, 1019, 1019, 1020, 891, 892, 893, 1104, 1105,
|
|
1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1072, 1073, 1074, 1075, 1076,
|
|
1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095,
|
|
1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082,
|
|
1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101,
|
|
1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121,
|
|
1121, 1123, 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137, 1137, 1139, 1139,
|
|
1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151, 1151, 1153, 1153, 1154, 1155, 1156, 1157, 1158,
|
|
1159, 1160, 1161, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173, 1173, 1175, 1175, 1177, 1177,
|
|
1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187, 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197,
|
|
1197, 1199, 1199, 1201, 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215, 1215,
|
|
1231, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230, 1230, 1231, 1233, 1233, 1235,
|
|
1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245, 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253,
|
|
1255, 1255, 1257, 1257, 1259, 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273,
|
|
1273, 1275, 1275, 1277, 1277, 1279, 1279};
|
|
|
|
static const int32 to_lower_ranges[] = {
|
|
1280, 2097153, 1328, 1328, 1329, -1378, 1367, -1368, 4256, -11521, 4294, 4294, 4295,
|
|
11559, 4296, -4297, 4301, 11565, 4302, -4303, 5024, -43889, 5104, -5113, 5110, -5111,
|
|
7680, 2097153, 7830, -7831, 7838, 223, 7839, 2097153, 7936, -7937, 7944, -7937, 7952,
|
|
-7953, 7960, -7953, 7966, -7967, 7976, -7969, 7984, -7985, 7992, -7985, 8000, -8001,
|
|
8008, -8001, 8014, -8015, 8025, 8017, 8026, 8026, 8027, 8019, 8028, 8028, 8029,
|
|
8021, 8030, 8030, 8031, 8023, 8032, -8033, 8040, -8033, 8048, -8049, 8072, -8065,
|
|
8080, -8081, 8088, -8081, 8096, -8097, 8104, -8097, 8112, -8113, 8120, -8113, 8122,
|
|
-8049, 8124, 8115, 8125, -8126, 8136, -8051, 8140, 8131, 8141, -8142, 8152, -8145,
|
|
8154, -8055, 8156, -8157, 8168, -8161, 8170, -8059, 8172, 8165, 8173, -8174, 8184,
|
|
-8057, 8186, -8061, 8188, 8179, 8189, -8190, 8486, 969, 8487, -8488, 8490, 107,
|
|
8491, 229, 8492, -8493, 8498, 8526, 8499, -8500, 8544, -8561, 8560, -8561, 8579,
|
|
8580, 8581, -8582, 9398, -9425, 9424, -9425, 11264, -11313, 11311, -11312, 11360, 11361,
|
|
11362, 619, 11363, 7549, 11364, 637, 11365, -11366, 11367, 11368, 11369, 11370, 11371,
|
|
11372, 11373, 593, 11374, 625, 11375, 592, 11376, 594, 11377, 2097153, 11380, 11380,
|
|
11381, 11382, 11383, -11384, 11390, -576, 11392, 2097153, 11492, -11493, 11499, 11500, 11501,
|
|
11502, 11503, -11504, 11506, 11507, 11508, -11509, 42560, 2097153, 42606, -42607, 42624, 2097153,
|
|
42652, -42653, 42786, 2097153, 42800, -42801, 42802, 2097153, 42864, -42865, 42873, 42874, 42875,
|
|
42876, 42877, 7545, 42878, 2097153, 42888, -42889, 42891, 42892, 42893, 613, 42894, -42895,
|
|
42896, 2097153, 42900, -42901, 42902, 2097153, 42922, 614, 42923, 604, 42924, 609, 42925,
|
|
620, 42926, 618, 42927, 42927, 42928, 670, 42929, 647, 42930, 669, 42931, 43859,
|
|
42932, 2097153, 42936, -42937, 65313, -65346, 65339, -65340, 66560, -66601, 66600, -66601, 66736,
|
|
-66777, 66772, -66773, 68736, -68801, 68787, -68788, 71840, -71873, 71872, -71873, 125184, -125219,
|
|
125218, -125219, 2147483647, 0};
|
|
|
|
UnicodeSimpleCategory get_unicode_simple_category(uint32 code) {
|
|
auto it = std::upper_bound(std::begin(unicode_simple_category_ranges), std::end(unicode_simple_category_ranges),
|
|
(code << 5) + 30);
|
|
return static_cast<UnicodeSimpleCategory>(*(it - 1) & 31);
|
|
}
|
|
|
|
/**
|
|
* Search pregenerated ranges of pairs for the replacement of specified character
|
|
*/
|
|
template <size_t N>
|
|
static uint32 binary_search_ranges(const int32 (&ranges)[N], uint32 code) {
|
|
if (code > 0x10ffff) {
|
|
return 0;
|
|
}
|
|
|
|
int32 code_int = static_cast<int32>(code);
|
|
size_t l = 0, r = N;
|
|
while (l < r) {
|
|
size_t m = ((l + r + 2) >> 2) << 1;
|
|
if (ranges[m] <= code_int) {
|
|
l = m;
|
|
} else {
|
|
r = m - 2;
|
|
}
|
|
}
|
|
|
|
int32 t = ranges[l + 1];
|
|
if (t < 0) {
|
|
return code - ranges[l] + (~t);
|
|
}
|
|
if (t <= 0x10ffff) {
|
|
return t;
|
|
}
|
|
switch (t - 0x200000) {
|
|
case 0:
|
|
return (code & -2);
|
|
case 1:
|
|
return (code | 1);
|
|
case 2:
|
|
return ((code - 1) | 1);
|
|
default:
|
|
UNREACHABLE();
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
uint32 prepare_search_character(uint32 code) {
|
|
if (code < TABLE_SIZE) {
|
|
return prepare_search_character_table[code];
|
|
} else {
|
|
return binary_search_ranges(prepare_search_character_ranges, code);
|
|
}
|
|
}
|
|
|
|
uint32 unicode_to_lower(uint32 code) {
|
|
if (code < TABLE_SIZE) {
|
|
return to_lower_table[code];
|
|
} else {
|
|
return binary_search_ranges(to_lower_ranges, code);
|
|
}
|
|
}
|
|
|
|
} // namespace td
|