code-points.js 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996
  1. 'use strict'
  2. const { range } = require('./util')
  3. /**
  4. * A.1 Unassigned code points in Unicode 3.2
  5. * @link https://tools.ietf.org/html/rfc3454#appendix-A.1
  6. */
  7. const unassigned_code_points = new Set([
  8. 0x0221,
  9. ...range(0x0234, 0x024F),
  10. ...range(0x02AE, 0x02AF),
  11. ...range(0x02EF, 0x02FF),
  12. ...range(0x0350, 0x035F),
  13. ...range(0x0370, 0x0373),
  14. ...range(0x0376, 0x0379),
  15. ...range(0x037B, 0x037D),
  16. ...range(0x037F, 0x0383),
  17. 0x038B,
  18. 0x038D,
  19. 0x03A2,
  20. 0x03CF,
  21. ...range(0x03F7, 0x03FF),
  22. 0x0487,
  23. 0x04CF,
  24. ...range(0x04F6, 0x04F7),
  25. ...range(0x04FA, 0x04FF),
  26. ...range(0x0510, 0x0530),
  27. ...range(0x0557, 0x0558),
  28. 0x0560,
  29. 0x0588,
  30. ...range(0x058B, 0x0590),
  31. 0x05A2,
  32. 0x05BA,
  33. ...range(0x05C5, 0x05CF),
  34. ...range(0x05EB, 0x05EF),
  35. ...range(0x05F5, 0x060B),
  36. ...range(0x060D, 0x061A),
  37. ...range(0x061C, 0x061E),
  38. 0x0620,
  39. ...range(0x063B, 0x063F),
  40. ...range(0x0656, 0x065F),
  41. ...range(0x06EE, 0x06EF),
  42. 0x06FF,
  43. 0x070E,
  44. ...range(0x072D, 0x072F),
  45. ...range(0x074B, 0x077F),
  46. ...range(0x07B2, 0x0900),
  47. 0x0904,
  48. ...range(0x093A, 0x093B),
  49. ...range(0x094E, 0x094F),
  50. ...range(0x0955, 0x0957),
  51. ...range(0x0971, 0x0980),
  52. 0x0984,
  53. ...range(0x098D, 0x098E),
  54. ...range(0x0991, 0x0992),
  55. 0x09A9,
  56. 0x09B1,
  57. ...range(0x09B3, 0x09B5),
  58. ...range(0x09BA, 0x09BB),
  59. 0x09BD,
  60. ...range(0x09C5, 0x09C6),
  61. ...range(0x09C9, 0x09CA),
  62. ...range(0x09CE, 0x09D6),
  63. ...range(0x09D8, 0x09DB),
  64. 0x09DE,
  65. ...range(0x09E4, 0x09E5),
  66. ...range(0x09FB, 0x0A01),
  67. ...range(0x0A03, 0x0A04),
  68. ...range(0x0A0B, 0x0A0E),
  69. ...range(0x0A11, 0x0A12),
  70. 0x0A29,
  71. 0x0A31,
  72. 0x0A34,
  73. 0x0A37,
  74. ...range(0x0A3A, 0x0A3B),
  75. 0x0A3D,
  76. ...range(0x0A43, 0x0A46),
  77. ...range(0x0A49, 0x0A4A),
  78. ...range(0x0A4E, 0x0A58),
  79. 0x0A5D,
  80. ...range(0x0A5F, 0x0A65),
  81. ...range(0x0A75, 0x0A80),
  82. 0x0A84,
  83. 0x0A8C,
  84. 0x0A8E,
  85. 0x0A92,
  86. 0x0AA9,
  87. 0x0AB1,
  88. 0x0AB4,
  89. ...range(0x0ABA, 0x0ABB),
  90. 0x0AC6,
  91. 0x0ACA,
  92. ...range(0x0ACE, 0x0ACF),
  93. ...range(0x0AD1, 0x0ADF),
  94. ...range(0x0AE1, 0x0AE5),
  95. ...range(0x0AF0, 0x0B00),
  96. 0x0B04,
  97. ...range(0x0B0D, 0x0B0E),
  98. ...range(0x0B11, 0x0B12),
  99. 0x0B29,
  100. 0x0B31,
  101. ...range(0x0B34, 0x0B35),
  102. ...range(0x0B3A, 0x0B3B),
  103. ...range(0x0B44, 0x0B46),
  104. ...range(0x0B49, 0x0B4A),
  105. ...range(0x0B4E, 0x0B55),
  106. ...range(0x0B58, 0x0B5B),
  107. 0x0B5E,
  108. ...range(0x0B62, 0x0B65),
  109. ...range(0x0B71, 0x0B81),
  110. 0x0B84,
  111. ...range(0x0B8B, 0x0B8D),
  112. 0x0B91,
  113. ...range(0x0B96, 0x0B98),
  114. 0x0B9B,
  115. 0x0B9D,
  116. ...range(0x0BA0, 0x0BA2),
  117. ...range(0x0BA5, 0x0BA7),
  118. ...range(0x0BAB, 0x0BAD),
  119. 0x0BB6,
  120. ...range(0x0BBA, 0x0BBD),
  121. ...range(0x0BC3, 0x0BC5),
  122. 0x0BC9,
  123. ...range(0x0BCE, 0x0BD6),
  124. ...range(0x0BD8, 0x0BE6),
  125. ...range(0x0BF3, 0x0C00),
  126. 0x0C04,
  127. 0x0C0D,
  128. 0x0C11,
  129. 0x0C29,
  130. 0x0C34,
  131. ...range(0x0C3A, 0x0C3D),
  132. 0x0C45,
  133. 0x0C49,
  134. ...range(0x0C4E, 0x0C54),
  135. ...range(0x0C57, 0x0C5F),
  136. ...range(0x0C62, 0x0C65),
  137. ...range(0x0C70, 0x0C81),
  138. 0x0C84,
  139. 0x0C8D,
  140. 0x0C91,
  141. 0x0CA9,
  142. 0x0CB4,
  143. ...range(0x0CBA, 0x0CBD),
  144. 0x0CC5,
  145. 0x0CC9,
  146. ...range(0x0CCE, 0x0CD4),
  147. ...range(0x0CD7, 0x0CDD),
  148. 0x0CDF,
  149. ...range(0x0CE2, 0x0CE5),
  150. ...range(0x0CF0, 0x0D01),
  151. 0x0D04,
  152. 0x0D0D,
  153. 0x0D11,
  154. 0x0D29,
  155. ...range(0x0D3A, 0x0D3D),
  156. ...range(0x0D44, 0x0D45),
  157. 0x0D49,
  158. ...range(0x0D4E, 0x0D56),
  159. ...range(0x0D58, 0x0D5F),
  160. ...range(0x0D62, 0x0D65),
  161. ...range(0x0D70, 0x0D81),
  162. 0x0D84,
  163. ...range(0x0D97, 0x0D99),
  164. 0x0DB2,
  165. 0x0DBC,
  166. ...range(0x0DBE, 0x0DBF),
  167. ...range(0x0DC7, 0x0DC9),
  168. ...range(0x0DCB, 0x0DCE),
  169. 0x0DD5,
  170. 0x0DD7,
  171. ...range(0x0DE0, 0x0DF1),
  172. ...range(0x0DF5, 0x0E00),
  173. ...range(0x0E3B, 0x0E3E),
  174. ...range(0x0E5C, 0x0E80),
  175. 0x0E83,
  176. ...range(0x0E85, 0x0E86),
  177. 0x0E89,
  178. ...range(0x0E8B, 0x0E8C),
  179. ...range(0x0E8E, 0x0E93),
  180. 0x0E98,
  181. 0x0EA0,
  182. 0x0EA4,
  183. 0x0EA6,
  184. ...range(0x0EA8, 0x0EA9),
  185. 0x0EAC,
  186. 0x0EBA,
  187. ...range(0x0EBE, 0x0EBF),
  188. 0x0EC5,
  189. 0x0EC7,
  190. ...range(0x0ECE, 0x0ECF),
  191. ...range(0x0EDA, 0x0EDB),
  192. ...range(0x0EDE, 0x0EFF),
  193. 0x0F48,
  194. ...range(0x0F6B, 0x0F70),
  195. ...range(0x0F8C, 0x0F8F),
  196. 0x0F98,
  197. 0x0FBD,
  198. ...range(0x0FCD, 0x0FCE),
  199. ...range(0x0FD0, 0x0FFF),
  200. 0x1022,
  201. 0x1028,
  202. 0x102B,
  203. ...range(0x1033, 0x1035),
  204. ...range(0x103A, 0x103F),
  205. ...range(0x105A, 0x109F),
  206. ...range(0x10C6, 0x10CF),
  207. ...range(0x10F9, 0x10FA),
  208. ...range(0x10FC, 0x10FF),
  209. ...range(0x115A, 0x115E),
  210. ...range(0x11A3, 0x11A7),
  211. ...range(0x11FA, 0x11FF),
  212. 0x1207,
  213. 0x1247,
  214. 0x1249,
  215. ...range(0x124E, 0x124F),
  216. 0x1257,
  217. 0x1259,
  218. ...range(0x125E, 0x125F),
  219. 0x1287,
  220. 0x1289,
  221. ...range(0x128E, 0x128F),
  222. 0x12AF,
  223. 0x12B1,
  224. ...range(0x12B6, 0x12B7),
  225. 0x12BF,
  226. 0x12C1,
  227. ...range(0x12C6, 0x12C7),
  228. 0x12CF,
  229. 0x12D7,
  230. 0x12EF,
  231. 0x130F,
  232. 0x1311,
  233. ...range(0x1316, 0x1317),
  234. 0x131F,
  235. 0x1347,
  236. ...range(0x135B, 0x1360),
  237. ...range(0x137D, 0x139F),
  238. ...range(0x13F5, 0x1400),
  239. ...range(0x1677, 0x167F),
  240. ...range(0x169D, 0x169F),
  241. ...range(0x16F1, 0x16FF),
  242. 0x170D,
  243. ...range(0x1715, 0x171F),
  244. ...range(0x1737, 0x173F),
  245. ...range(0x1754, 0x175F),
  246. 0x176D,
  247. 0x1771,
  248. ...range(0x1774, 0x177F),
  249. ...range(0x17DD, 0x17DF),
  250. ...range(0x17EA, 0x17FF),
  251. 0x180F,
  252. ...range(0x181A, 0x181F),
  253. ...range(0x1878, 0x187F),
  254. ...range(0x18AA, 0x1DFF),
  255. ...range(0x1E9C, 0x1E9F),
  256. ...range(0x1EFA, 0x1EFF),
  257. ...range(0x1F16, 0x1F17),
  258. ...range(0x1F1E, 0x1F1F),
  259. ...range(0x1F46, 0x1F47),
  260. ...range(0x1F4E, 0x1F4F),
  261. 0x1F58,
  262. 0x1F5A,
  263. 0x1F5C,
  264. 0x1F5E,
  265. ...range(0x1F7E, 0x1F7F),
  266. 0x1FB5,
  267. 0x1FC5,
  268. ...range(0x1FD4, 0x1FD5),
  269. 0x1FDC,
  270. ...range(0x1FF0, 0x1FF1),
  271. 0x1FF5,
  272. 0x1FFF,
  273. ...range(0x2053, 0x2056),
  274. ...range(0x2058, 0x205E),
  275. ...range(0x2064, 0x2069),
  276. ...range(0x2072, 0x2073),
  277. ...range(0x208F, 0x209F),
  278. ...range(0x20B2, 0x20CF),
  279. ...range(0x20EB, 0x20FF),
  280. ...range(0x213B, 0x213C),
  281. ...range(0x214C, 0x2152),
  282. ...range(0x2184, 0x218F),
  283. ...range(0x23CF, 0x23FF),
  284. ...range(0x2427, 0x243F),
  285. ...range(0x244B, 0x245F),
  286. 0x24FF,
  287. ...range(0x2614, 0x2615),
  288. 0x2618,
  289. ...range(0x267E, 0x267F),
  290. ...range(0x268A, 0x2700),
  291. 0x2705,
  292. ...range(0x270A, 0x270B),
  293. 0x2728,
  294. 0x274C,
  295. 0x274E,
  296. ...range(0x2753, 0x2755),
  297. 0x2757,
  298. ...range(0x275F, 0x2760),
  299. ...range(0x2795, 0x2797),
  300. 0x27B0,
  301. ...range(0x27BF, 0x27CF),
  302. ...range(0x27EC, 0x27EF),
  303. ...range(0x2B00, 0x2E7F),
  304. 0x2E9A,
  305. ...range(0x2EF4, 0x2EFF),
  306. ...range(0x2FD6, 0x2FEF),
  307. ...range(0x2FFC, 0x2FFF),
  308. 0x3040,
  309. ...range(0x3097, 0x3098),
  310. ...range(0x3100, 0x3104),
  311. ...range(0x312D, 0x3130),
  312. 0x318F,
  313. ...range(0x31B8, 0x31EF),
  314. ...range(0x321D, 0x321F),
  315. ...range(0x3244, 0x3250),
  316. ...range(0x327C, 0x327E),
  317. ...range(0x32CC, 0x32CF),
  318. 0x32FF,
  319. ...range(0x3377, 0x337A),
  320. ...range(0x33DE, 0x33DF),
  321. 0x33FF,
  322. ...range(0x4DB6, 0x4DFF),
  323. ...range(0x9FA6, 0x9FFF),
  324. ...range(0xA48D, 0xA48F),
  325. ...range(0xA4C7, 0xABFF),
  326. ...range(0xD7A4, 0xD7FF),
  327. ...range(0xFA2E, 0xFA2F),
  328. ...range(0xFA6B, 0xFAFF),
  329. ...range(0xFB07, 0xFB12),
  330. ...range(0xFB18, 0xFB1C),
  331. 0xFB37,
  332. 0xFB3D,
  333. 0xFB3F,
  334. 0xFB42,
  335. 0xFB45,
  336. ...range(0xFBB2, 0xFBD2),
  337. ...range(0xFD40, 0xFD4F),
  338. ...range(0xFD90, 0xFD91),
  339. ...range(0xFDC8, 0xFDCF),
  340. ...range(0xFDFD, 0xFDFF),
  341. ...range(0xFE10, 0xFE1F),
  342. ...range(0xFE24, 0xFE2F),
  343. ...range(0xFE47, 0xFE48),
  344. 0xFE53,
  345. 0xFE67,
  346. ...range(0xFE6C, 0xFE6F),
  347. 0xFE75,
  348. ...range(0xFEFD, 0xFEFE),
  349. 0xFF00,
  350. ...range(0xFFBF, 0xFFC1),
  351. ...range(0xFFC8, 0xFFC9),
  352. ...range(0xFFD0, 0xFFD1),
  353. ...range(0xFFD8, 0xFFD9),
  354. ...range(0xFFDD, 0xFFDF),
  355. 0xFFE7,
  356. ...range(0xFFEF, 0xFFF8),
  357. ...range(0x10000, 0x102FF),
  358. 0x1031F,
  359. ...range(0x10324, 0x1032F),
  360. ...range(0x1034B, 0x103FF),
  361. ...range(0x10426, 0x10427),
  362. ...range(0x1044E, 0x1CFFF),
  363. ...range(0x1D0F6, 0x1D0FF),
  364. ...range(0x1D127, 0x1D129),
  365. ...range(0x1D1DE, 0x1D3FF),
  366. 0x1D455,
  367. 0x1D49D,
  368. ...range(0x1D4A0, 0x1D4A1),
  369. ...range(0x1D4A3, 0x1D4A4),
  370. ...range(0x1D4A7, 0x1D4A8),
  371. 0x1D4AD,
  372. 0x1D4BA,
  373. 0x1D4BC,
  374. 0x1D4C1,
  375. 0x1D4C4,
  376. 0x1D506,
  377. ...range(0x1D50B, 0x1D50C),
  378. 0x1D515,
  379. 0x1D51D,
  380. 0x1D53A,
  381. 0x1D53F,
  382. 0x1D545,
  383. ...range(0x1D547, 0x1D549),
  384. 0x1D551,
  385. ...range(0x1D6A4, 0x1D6A7),
  386. ...range(0x1D7CA, 0x1D7CD),
  387. ...range(0x1D800, 0x1FFFD),
  388. ...range(0x2A6D7, 0x2F7FF),
  389. ...range(0x2FA1E, 0x2FFFD),
  390. ...range(0x30000, 0x3FFFD),
  391. ...range(0x40000, 0x4FFFD),
  392. ...range(0x50000, 0x5FFFD),
  393. ...range(0x60000, 0x6FFFD),
  394. ...range(0x70000, 0x7FFFD),
  395. ...range(0x80000, 0x8FFFD),
  396. ...range(0x90000, 0x9FFFD),
  397. ...range(0xA0000, 0xAFFFD),
  398. ...range(0xB0000, 0xBFFFD),
  399. ...range(0xC0000, 0xCFFFD),
  400. ...range(0xD0000, 0xDFFFD),
  401. 0xE0000,
  402. ...range(0xE0002, 0xE001F),
  403. ...range(0xE0080, 0xEFFFD),
  404. ])
  405. /**
  406. * B.1 Commonly mapped to nothing
  407. * @link https://tools.ietf.org/html/rfc3454#appendix-B.1
  408. */
  409. const commonly_mapped_to_nothing = new Set([
  410. 0x00AD,
  411. 0x034F,
  412. 0x1806,
  413. 0x180B,
  414. 0x180C,
  415. 0x180D,
  416. 0x200B,
  417. 0x200C,
  418. 0x200D,
  419. 0x2060,
  420. 0xFE00,
  421. 0xFE01,
  422. 0xFE02,
  423. 0xFE03,
  424. 0xFE04,
  425. 0xFE05,
  426. 0xFE06,
  427. 0xFE07,
  428. 0xFE08,
  429. 0xFE09,
  430. 0xFE0A,
  431. 0xFE0B,
  432. 0xFE0C,
  433. 0xFE0D,
  434. 0xFE0E,
  435. 0xFE0F,
  436. 0xFEFF,
  437. ])
  438. /**
  439. * C.1.2 Non-ASCII space characters
  440. * @link https://tools.ietf.org/html/rfc3454#appendix-C.1.2
  441. */
  442. const non_ASCII_space_characters = new Set([
  443. 0x00a0 /* NO-BREAK SPACE */,
  444. 0x1680 /* OGHAM SPACE MARK */,
  445. 0x2000 /* EN QUAD */,
  446. 0x2001 /* EM QUAD */,
  447. 0x2002 /* EN SPACE */,
  448. 0x2003 /* EM SPACE */,
  449. 0x2004 /* THREE-PER-EM SPACE */,
  450. 0x2005 /* FOUR-PER-EM SPACE */,
  451. 0x2006 /* SIX-PER-EM SPACE */,
  452. 0x2007 /* FIGURE SPACE */,
  453. 0x2008 /* PUNCTUATION SPACE */,
  454. 0x2009 /* THIN SPACE */,
  455. 0x200a /* HAIR SPACE */,
  456. 0x200b /* ZERO WIDTH SPACE */,
  457. 0x202f /* NARROW NO-BREAK SPACE */,
  458. 0x205f /* MEDIUM MATHEMATICAL SPACE */,
  459. 0x3000 /* IDEOGRAPHIC SPACE */,
  460. ])
  461. /**
  462. * 2.3. Prohibited Output
  463. * @type {Set}
  464. */
  465. const prohibited_characters = new Set([
  466. ...non_ASCII_space_characters,
  467. /**
  468. * C.2.1 ASCII control characters
  469. * @link https://tools.ietf.org/html/rfc3454#appendix-C.2.1
  470. */
  471. ...range(0, 0x001f) /* [CONTROL CHARACTERS] */,
  472. 0x007f /* DELETE */,
  473. /**
  474. * C.2.2 Non-ASCII control characters
  475. * @link https://tools.ietf.org/html/rfc3454#appendix-C.2.2
  476. */
  477. ...range(0x0080, 0x009F), /* [CONTROL CHARACTERS] */
  478. 0x06DD, /* ARABIC END OF AYAH */
  479. 0x070F, /* SYRIAC ABBREVIATION MARK */
  480. 0x180E, /* MONGOLIAN VOWEL SEPARATOR */
  481. 0x200C, /* ZERO WIDTH NON-JOINER */
  482. 0x200D, /* ZERO WIDTH JOINER */
  483. 0x2028, /* LINE SEPARATOR */
  484. 0x2029, /* PARAGRAPH SEPARATOR */
  485. 0x2060, /* WORD JOINER */
  486. 0x2061, /* FUNCTION APPLICATION */
  487. 0x2062, /* INVISIBLE TIMES */
  488. 0x2063, /* INVISIBLE SEPARATOR */
  489. ...range(0x206A, 0x206F), /* [CONTROL CHARACTERS] */
  490. 0xFEFF, /* ZERO WIDTH NO-BREAK SPACE */
  491. ...range(0xFFF9, 0xFFFC), /* [CONTROL CHARACTERS] */
  492. ...range(0x1D173, 0x1D17A), /* [MUSICAL CONTROL CHARACTERS] */
  493. /**
  494. * C.3 Private use
  495. * @link https://tools.ietf.org/html/rfc3454#appendix-C.3
  496. */
  497. ...range(0xE000, 0xF8FF), /* [PRIVATE USE, PLANE 0] */
  498. ...range(0xF0000, 0xFFFFD), /* [PRIVATE USE, PLANE 15] */
  499. ...range(0x100000, 0x10FFFD), /* [PRIVATE USE, PLANE 16] */
  500. /**
  501. * C.4 Non-character code points
  502. * @link https://tools.ietf.org/html/rfc3454#appendix-C.4
  503. */
  504. ...range(0xFDD0, 0xFDEF), /* [NONCHARACTER CODE POINTS] */
  505. ...range(0xFFFE, 0xFFFF), /* [NONCHARACTER CODE POINTS] */
  506. ...range(0x1FFFE, 0x1FFFF), /* [NONCHARACTER CODE POINTS] */
  507. ...range(0x2FFFE, 0x2FFFF), /* [NONCHARACTER CODE POINTS] */
  508. ...range(0x3FFFE, 0x3FFFF), /* [NONCHARACTER CODE POINTS] */
  509. ...range(0x4FFFE, 0x4FFFF), /* [NONCHARACTER CODE POINTS] */
  510. ...range(0x5FFFE, 0x5FFFF), /* [NONCHARACTER CODE POINTS] */
  511. ...range(0x6FFFE, 0x6FFFF), /* [NONCHARACTER CODE POINTS] */
  512. ...range(0x7FFFE, 0x7FFFF), /* [NONCHARACTER CODE POINTS] */
  513. ...range(0x8FFFE, 0x8FFFF), /* [NONCHARACTER CODE POINTS] */
  514. ...range(0x9FFFE, 0x9FFFF), /* [NONCHARACTER CODE POINTS] */
  515. ...range(0xAFFFE, 0xAFFFF), /* [NONCHARACTER CODE POINTS] */
  516. ...range(0xBFFFE, 0xBFFFF), /* [NONCHARACTER CODE POINTS] */
  517. ...range(0xCFFFE, 0xCFFFF), /* [NONCHARACTER CODE POINTS] */
  518. ...range(0xDFFFE, 0xDFFFF), /* [NONCHARACTER CODE POINTS] */
  519. ...range(0xEFFFE, 0xEFFFF), /* [NONCHARACTER CODE POINTS] */
  520. ...range(0x10FFFE, 0x10FFFF), /* [NONCHARACTER CODE POINTS] */
  521. /**
  522. * C.5 Surrogate codes
  523. * @link https://tools.ietf.org/html/rfc3454#appendix-C.5
  524. */
  525. ...range(0xD800, 0xDFFF),
  526. /**
  527. * C.6 Inappropriate for plain text
  528. * @link https://tools.ietf.org/html/rfc3454#appendix-C.6
  529. */
  530. 0xFFF9, /* INTERLINEAR ANNOTATION ANCHOR */
  531. 0xFFFA, /* INTERLINEAR ANNOTATION SEPARATOR */
  532. 0xFFFB, /* INTERLINEAR ANNOTATION TERMINATOR */
  533. 0xFFFC, /* OBJECT REPLACEMENT CHARACTER */
  534. 0xFFFD, /* REPLACEMENT CHARACTER */
  535. /**
  536. * C.7 Inappropriate for canonical representation
  537. * @link https://tools.ietf.org/html/rfc3454#appendix-C.7
  538. */
  539. ...range(0x2FF0, 0x2FFB), /* [IDEOGRAPHIC DESCRIPTION CHARACTERS] */
  540. /**
  541. * C.8 Change display properties or are deprecated
  542. * @link https://tools.ietf.org/html/rfc3454#appendix-C.8
  543. */
  544. 0x0340, /* COMBINING GRAVE TONE MARK */
  545. 0x0341, /* COMBINING ACUTE TONE MARK */
  546. 0x200E, /* LEFT-TO-RIGHT MARK */
  547. 0x200F, /* RIGHT-TO-LEFT MARK */
  548. 0x202A, /* LEFT-TO-RIGHT EMBEDDING */
  549. 0x202B, /* RIGHT-TO-LEFT EMBEDDING */
  550. 0x202C, /* POP DIRECTIONAL FORMATTING */
  551. 0x202D, /* LEFT-TO-RIGHT OVERRIDE */
  552. 0x202E, /* RIGHT-TO-LEFT OVERRIDE */
  553. 0x206A, /* INHIBIT SYMMETRIC SWAPPING */
  554. 0x206B, /* ACTIVATE SYMMETRIC SWAPPING */
  555. 0x206C, /* INHIBIT ARABIC FORM SHAPING */
  556. 0x206D, /* ACTIVATE ARABIC FORM SHAPING */
  557. 0x206E, /* NATIONAL DIGIT SHAPES */
  558. 0x206F, /* NOMINAL DIGIT SHAPES */
  559. /**
  560. * C.9 Tagging characters
  561. * @link https://tools.ietf.org/html/rfc3454#appendix-C.9
  562. */
  563. 0xE0001, /* LANGUAGE TAG */
  564. ...range(0xE0020, 0xE007F) /* [TAGGING CHARACTERS] */
  565. ])
  566. /**
  567. * D.1 Characters with bidirectional property "R" or "AL"
  568. * @link https://tools.ietf.org/html/rfc3454#appendix-D.1
  569. */
  570. const bidirectional_r_al = new Set([
  571. 0x05BE,
  572. 0x05C0,
  573. 0x05C3,
  574. ...range(0x05D0, 0x05EA),
  575. ...range(0x05F0, 0x05F4),
  576. 0x061B,
  577. 0x061F,
  578. ...range(0x0621, 0x063A),
  579. ...range(0x0640, 0x064A),
  580. ...range(0x066D, 0x066F),
  581. ...range(0x0671, 0x06D5),
  582. 0x06DD,
  583. ...range(0x06E5, 0x06E6),
  584. ...range(0x06FA, 0x06FE),
  585. ...range(0x0700, 0x070D),
  586. 0x0710,
  587. ...range(0x0712, 0x072C),
  588. ...range(0x0780, 0x07A5),
  589. 0x07B1,
  590. 0x200F,
  591. 0xFB1D,
  592. ...range(0xFB1F, 0xFB28),
  593. ...range(0xFB2A, 0xFB36),
  594. ...range(0xFB38, 0xFB3C),
  595. 0xFB3E,
  596. ...range(0xFB40, 0xFB41),
  597. ...range(0xFB43, 0xFB44),
  598. ...range(0xFB46, 0xFBB1),
  599. ...range(0xFBD3, 0xFD3D),
  600. ...range(0xFD50, 0xFD8F),
  601. ...range(0xFD92, 0xFDC7),
  602. ...range(0xFDF0, 0xFDFC),
  603. ...range(0xFE70, 0xFE74),
  604. ...range(0xFE76, 0xFEFC),
  605. ])
  606. /**
  607. * D.2 Characters with bidirectional property "L"
  608. * @link https://tools.ietf.org/html/rfc3454#appendix-D.2
  609. */
  610. const bidirectional_l = new Set([
  611. ...range(0x0041, 0x005A),
  612. ...range(0x0061, 0x007A),
  613. 0x00AA,
  614. 0x00B5,
  615. 0x00BA,
  616. ...range(0x00C0, 0x00D6),
  617. ...range(0x00D8, 0x00F6),
  618. ...range(0x00F8, 0x0220),
  619. ...range(0x0222, 0x0233),
  620. ...range(0x0250, 0x02AD),
  621. ...range(0x02B0, 0x02B8),
  622. ...range(0x02BB, 0x02C1),
  623. ...range(0x02D0, 0x02D1),
  624. ...range(0x02E0, 0x02E4),
  625. 0x02EE,
  626. 0x037A,
  627. 0x0386,
  628. ...range(0x0388, 0x038A),
  629. 0x038C,
  630. ...range(0x038E, 0x03A1),
  631. ...range(0x03A3, 0x03CE),
  632. ...range(0x03D0, 0x03F5),
  633. ...range(0x0400, 0x0482),
  634. ...range(0x048A, 0x04CE),
  635. ...range(0x04D0, 0x04F5),
  636. ...range(0x04F8, 0x04F9),
  637. ...range(0x0500, 0x050F),
  638. ...range(0x0531, 0x0556),
  639. ...range(0x0559, 0x055F),
  640. ...range(0x0561, 0x0587),
  641. 0x0589,
  642. 0x0903,
  643. ...range(0x0905, 0x0939),
  644. ...range(0x093D, 0x0940),
  645. ...range(0x0949, 0x094C),
  646. 0x0950,
  647. ...range(0x0958, 0x0961),
  648. ...range(0x0964, 0x0970),
  649. ...range(0x0982, 0x0983),
  650. ...range(0x0985, 0x098C),
  651. ...range(0x098F, 0x0990),
  652. ...range(0x0993, 0x09A8),
  653. ...range(0x09AA, 0x09B0),
  654. 0x09B2,
  655. ...range(0x09B6, 0x09B9),
  656. ...range(0x09BE, 0x09C0),
  657. ...range(0x09C7, 0x09C8),
  658. ...range(0x09CB, 0x09CC),
  659. 0x09D7,
  660. ...range(0x09DC, 0x09DD),
  661. ...range(0x09DF, 0x09E1),
  662. ...range(0x09E6, 0x09F1),
  663. ...range(0x09F4, 0x09FA),
  664. ...range(0x0A05, 0x0A0A),
  665. ...range(0x0A0F, 0x0A10),
  666. ...range(0x0A13, 0x0A28),
  667. ...range(0x0A2A, 0x0A30),
  668. ...range(0x0A32, 0x0A33),
  669. ...range(0x0A35, 0x0A36),
  670. ...range(0x0A38, 0x0A39),
  671. ...range(0x0A3E, 0x0A40),
  672. ...range(0x0A59, 0x0A5C),
  673. 0x0A5E,
  674. ...range(0x0A66, 0x0A6F),
  675. ...range(0x0A72, 0x0A74),
  676. 0x0A83,
  677. ...range(0x0A85, 0x0A8B),
  678. 0x0A8D,
  679. ...range(0x0A8F, 0x0A91),
  680. ...range(0x0A93, 0x0AA8),
  681. ...range(0x0AAA, 0x0AB0),
  682. ...range(0x0AB2, 0x0AB3),
  683. ...range(0x0AB5, 0x0AB9),
  684. ...range(0x0ABD, 0x0AC0),
  685. 0x0AC9,
  686. ...range(0x0ACB, 0x0ACC),
  687. 0x0AD0,
  688. 0x0AE0,
  689. ...range(0x0AE6, 0x0AEF),
  690. ...range(0x0B02, 0x0B03),
  691. ...range(0x0B05, 0x0B0C),
  692. ...range(0x0B0F, 0x0B10),
  693. ...range(0x0B13, 0x0B28),
  694. ...range(0x0B2A, 0x0B30),
  695. ...range(0x0B32, 0x0B33),
  696. ...range(0x0B36, 0x0B39),
  697. ...range(0x0B3D, 0x0B3E),
  698. 0x0B40,
  699. ...range(0x0B47, 0x0B48),
  700. ...range(0x0B4B, 0x0B4C),
  701. 0x0B57,
  702. ...range(0x0B5C, 0x0B5D),
  703. ...range(0x0B5F, 0x0B61),
  704. ...range(0x0B66, 0x0B70),
  705. 0x0B83,
  706. ...range(0x0B85, 0x0B8A),
  707. ...range(0x0B8E, 0x0B90),
  708. ...range(0x0B92, 0x0B95),
  709. ...range(0x0B99, 0x0B9A),
  710. 0x0B9C,
  711. ...range(0x0B9E, 0x0B9F),
  712. ...range(0x0BA3, 0x0BA4),
  713. ...range(0x0BA8, 0x0BAA),
  714. ...range(0x0BAE, 0x0BB5),
  715. ...range(0x0BB7, 0x0BB9),
  716. ...range(0x0BBE, 0x0BBF),
  717. ...range(0x0BC1, 0x0BC2),
  718. ...range(0x0BC6, 0x0BC8),
  719. ...range(0x0BCA, 0x0BCC),
  720. 0x0BD7,
  721. ...range(0x0BE7, 0x0BF2),
  722. ...range(0x0C01, 0x0C03),
  723. ...range(0x0C05, 0x0C0C),
  724. ...range(0x0C0E, 0x0C10),
  725. ...range(0x0C12, 0x0C28),
  726. ...range(0x0C2A, 0x0C33),
  727. ...range(0x0C35, 0x0C39),
  728. ...range(0x0C41, 0x0C44),
  729. ...range(0x0C60, 0x0C61),
  730. ...range(0x0C66, 0x0C6F),
  731. ...range(0x0C82, 0x0C83),
  732. ...range(0x0C85, 0x0C8C),
  733. ...range(0x0C8E, 0x0C90),
  734. ...range(0x0C92, 0x0CA8),
  735. ...range(0x0CAA, 0x0CB3),
  736. ...range(0x0CB5, 0x0CB9),
  737. 0x0CBE,
  738. ...range(0x0CC0, 0x0CC4),
  739. ...range(0x0CC7, 0x0CC8),
  740. ...range(0x0CCA, 0x0CCB),
  741. ...range(0x0CD5, 0x0CD6),
  742. 0x0CDE,
  743. ...range(0x0CE0, 0x0CE1),
  744. ...range(0x0CE6, 0x0CEF),
  745. ...range(0x0D02, 0x0D03),
  746. ...range(0x0D05, 0x0D0C),
  747. ...range(0x0D0E, 0x0D10),
  748. ...range(0x0D12, 0x0D28),
  749. ...range(0x0D2A, 0x0D39),
  750. ...range(0x0D3E, 0x0D40),
  751. ...range(0x0D46, 0x0D48),
  752. ...range(0x0D4A, 0x0D4C),
  753. 0x0D57,
  754. ...range(0x0D60, 0x0D61),
  755. ...range(0x0D66, 0x0D6F),
  756. ...range(0x0D82, 0x0D83),
  757. ...range(0x0D85, 0x0D96),
  758. ...range(0x0D9A, 0x0DB1),
  759. ...range(0x0DB3, 0x0DBB),
  760. 0x0DBD,
  761. ...range(0x0DC0, 0x0DC6),
  762. ...range(0x0DCF, 0x0DD1),
  763. ...range(0x0DD8, 0x0DDF),
  764. ...range(0x0DF2, 0x0DF4),
  765. ...range(0x0E01, 0x0E30),
  766. ...range(0x0E32, 0x0E33),
  767. ...range(0x0E40, 0x0E46),
  768. ...range(0x0E4F, 0x0E5B),
  769. ...range(0x0E81, 0x0E82),
  770. 0x0E84,
  771. ...range(0x0E87, 0x0E88),
  772. 0x0E8A,
  773. 0x0E8D,
  774. ...range(0x0E94, 0x0E97),
  775. ...range(0x0E99, 0x0E9F),
  776. ...range(0x0EA1, 0x0EA3),
  777. 0x0EA5,
  778. 0x0EA7,
  779. ...range(0x0EAA, 0x0EAB),
  780. ...range(0x0EAD, 0x0EB0),
  781. ...range(0x0EB2, 0x0EB3),
  782. 0x0EBD,
  783. ...range(0x0EC0, 0x0EC4),
  784. 0x0EC6,
  785. ...range(0x0ED0, 0x0ED9),
  786. ...range(0x0EDC, 0x0EDD),
  787. ...range(0x0F00, 0x0F17),
  788. ...range(0x0F1A, 0x0F34),
  789. 0x0F36,
  790. 0x0F38,
  791. ...range(0x0F3E, 0x0F47),
  792. ...range(0x0F49, 0x0F6A),
  793. 0x0F7F,
  794. 0x0F85,
  795. ...range(0x0F88, 0x0F8B),
  796. ...range(0x0FBE, 0x0FC5),
  797. ...range(0x0FC7, 0x0FCC),
  798. 0x0FCF,
  799. ...range(0x1000, 0x1021),
  800. ...range(0x1023, 0x1027),
  801. ...range(0x1029, 0x102A),
  802. 0x102C,
  803. 0x1031,
  804. 0x1038,
  805. ...range(0x1040, 0x1057),
  806. ...range(0x10A0, 0x10C5),
  807. ...range(0x10D0, 0x10F8),
  808. 0x10FB,
  809. ...range(0x1100, 0x1159),
  810. ...range(0x115F, 0x11A2),
  811. ...range(0x11A8, 0x11F9),
  812. ...range(0x1200, 0x1206),
  813. ...range(0x1208, 0x1246),
  814. 0x1248,
  815. ...range(0x124A, 0x124D),
  816. ...range(0x1250, 0x1256),
  817. 0x1258,
  818. ...range(0x125A, 0x125D),
  819. ...range(0x1260, 0x1286),
  820. 0x1288,
  821. ...range(0x128A, 0x128D),
  822. ...range(0x1290, 0x12AE),
  823. 0x12B0,
  824. ...range(0x12B2, 0x12B5),
  825. ...range(0x12B8, 0x12BE),
  826. 0x12C0,
  827. ...range(0x12C2, 0x12C5),
  828. ...range(0x12C8, 0x12CE),
  829. ...range(0x12D0, 0x12D6),
  830. ...range(0x12D8, 0x12EE),
  831. ...range(0x12F0, 0x130E),
  832. 0x1310,
  833. ...range(0x1312, 0x1315),
  834. ...range(0x1318, 0x131E),
  835. ...range(0x1320, 0x1346),
  836. ...range(0x1348, 0x135A),
  837. ...range(0x1361, 0x137C),
  838. ...range(0x13A0, 0x13F4),
  839. ...range(0x1401, 0x1676),
  840. ...range(0x1681, 0x169A),
  841. ...range(0x16A0, 0x16F0),
  842. ...range(0x1700, 0x170C),
  843. ...range(0x170E, 0x1711),
  844. ...range(0x1720, 0x1731),
  845. ...range(0x1735, 0x1736),
  846. ...range(0x1740, 0x1751),
  847. ...range(0x1760, 0x176C),
  848. ...range(0x176E, 0x1770),
  849. ...range(0x1780, 0x17B6),
  850. ...range(0x17BE, 0x17C5),
  851. ...range(0x17C7, 0x17C8),
  852. ...range(0x17D4, 0x17DA),
  853. 0x17DC,
  854. ...range(0x17E0, 0x17E9),
  855. ...range(0x1810, 0x1819),
  856. ...range(0x1820, 0x1877),
  857. ...range(0x1880, 0x18A8),
  858. ...range(0x1E00, 0x1E9B),
  859. ...range(0x1EA0, 0x1EF9),
  860. ...range(0x1F00, 0x1F15),
  861. ...range(0x1F18, 0x1F1D),
  862. ...range(0x1F20, 0x1F45),
  863. ...range(0x1F48, 0x1F4D),
  864. ...range(0x1F50, 0x1F57),
  865. 0x1F59,
  866. 0x1F5B,
  867. 0x1F5D,
  868. ...range(0x1F5F, 0x1F7D),
  869. ...range(0x1F80, 0x1FB4),
  870. ...range(0x1FB6, 0x1FBC),
  871. 0x1FBE,
  872. ...range(0x1FC2, 0x1FC4),
  873. ...range(0x1FC6, 0x1FCC),
  874. ...range(0x1FD0, 0x1FD3),
  875. ...range(0x1FD6, 0x1FDB),
  876. ...range(0x1FE0, 0x1FEC),
  877. ...range(0x1FF2, 0x1FF4),
  878. ...range(0x1FF6, 0x1FFC),
  879. 0x200E,
  880. 0x2071,
  881. 0x207F,
  882. 0x2102,
  883. 0x2107,
  884. ...range(0x210A, 0x2113),
  885. 0x2115,
  886. ...range(0x2119, 0x211D),
  887. 0x2124,
  888. 0x2126,
  889. 0x2128,
  890. ...range(0x212A, 0x212D),
  891. ...range(0x212F, 0x2131),
  892. ...range(0x2133, 0x2139),
  893. ...range(0x213D, 0x213F),
  894. ...range(0x2145, 0x2149),
  895. ...range(0x2160, 0x2183),
  896. ...range(0x2336, 0x237A),
  897. 0x2395,
  898. ...range(0x249C, 0x24E9),
  899. ...range(0x3005, 0x3007),
  900. ...range(0x3021, 0x3029),
  901. ...range(0x3031, 0x3035),
  902. ...range(0x3038, 0x303C),
  903. ...range(0x3041, 0x3096),
  904. ...range(0x309D, 0x309F),
  905. ...range(0x30A1, 0x30FA),
  906. ...range(0x30FC, 0x30FF),
  907. ...range(0x3105, 0x312C),
  908. ...range(0x3131, 0x318E),
  909. ...range(0x3190, 0x31B7),
  910. ...range(0x31F0, 0x321C),
  911. ...range(0x3220, 0x3243),
  912. ...range(0x3260, 0x327B),
  913. ...range(0x327F, 0x32B0),
  914. ...range(0x32C0, 0x32CB),
  915. ...range(0x32D0, 0x32FE),
  916. ...range(0x3300, 0x3376),
  917. ...range(0x337B, 0x33DD),
  918. ...range(0x33E0, 0x33FE),
  919. ...range(0x3400, 0x4DB5),
  920. ...range(0x4E00, 0x9FA5),
  921. ...range(0xA000, 0xA48C),
  922. ...range(0xAC00, 0xD7A3),
  923. ...range(0xD800, 0xFA2D),
  924. ...range(0xFA30, 0xFA6A),
  925. ...range(0xFB00, 0xFB06),
  926. ...range(0xFB13, 0xFB17),
  927. ...range(0xFF21, 0xFF3A),
  928. ...range(0xFF41, 0xFF5A),
  929. ...range(0xFF66, 0xFFBE),
  930. ...range(0xFFC2, 0xFFC7),
  931. ...range(0xFFCA, 0xFFCF),
  932. ...range(0xFFD2, 0xFFD7),
  933. ...range(0xFFDA, 0xFFDC),
  934. ...range(0x10300, 0x1031E),
  935. ...range(0x10320, 0x10323),
  936. ...range(0x10330, 0x1034A),
  937. ...range(0x10400, 0x10425),
  938. ...range(0x10428, 0x1044D),
  939. ...range(0x1D000, 0x1D0F5),
  940. ...range(0x1D100, 0x1D126),
  941. ...range(0x1D12A, 0x1D166),
  942. ...range(0x1D16A, 0x1D172),
  943. ...range(0x1D183, 0x1D184),
  944. ...range(0x1D18C, 0x1D1A9),
  945. ...range(0x1D1AE, 0x1D1DD),
  946. ...range(0x1D400, 0x1D454),
  947. ...range(0x1D456, 0x1D49C),
  948. ...range(0x1D49E, 0x1D49F),
  949. 0x1D4A2,
  950. ...range(0x1D4A5, 0x1D4A6),
  951. ...range(0x1D4A9, 0x1D4AC),
  952. ...range(0x1D4AE, 0x1D4B9),
  953. 0x1D4BB,
  954. ...range(0x1D4BD, 0x1D4C0),
  955. ...range(0x1D4C2, 0x1D4C3),
  956. ...range(0x1D4C5, 0x1D505),
  957. ...range(0x1D507, 0x1D50A),
  958. ...range(0x1D50D, 0x1D514),
  959. ...range(0x1D516, 0x1D51C),
  960. ...range(0x1D51E, 0x1D539),
  961. ...range(0x1D53B, 0x1D53E),
  962. ...range(0x1D540, 0x1D544),
  963. 0x1D546,
  964. ...range(0x1D54A, 0x1D550),
  965. ...range(0x1D552, 0x1D6A3),
  966. ...range(0x1D6A8, 0x1D7C9),
  967. ...range(0x20000, 0x2A6D6),
  968. ...range(0x2F800, 0x2FA1D),
  969. ...range(0xF0000, 0xFFFFD),
  970. ...range(0x100000, 0x10FFFD)
  971. ])
  972. module.exports = {
  973. unassigned_code_points,
  974. commonly_mapped_to_nothing,
  975. non_ASCII_space_characters,
  976. prohibited_characters,
  977. bidirectional_r_al,
  978. bidirectional_l
  979. }