pax_global_header00006660000000000000000000000064152611367270014524gustar00rootroot0000000000000052 comment=8bfec9063cc840b1bda0c4de7fab9670a467fce6 sedlex-3.8.1/000077500000000000000000000000001526113672700130215ustar00rootroot00000000000000sedlex-3.8.1/.gitattributes000066400000000000000000000000361526113672700157130ustar00rootroot00000000000000test/stress_test.ml text diff sedlex-3.8.1/.github/000077500000000000000000000000001526113672700143615ustar00rootroot00000000000000sedlex-3.8.1/.github/CODEOWNERS000066400000000000000000000002411526113672700157510ustar00rootroot00000000000000# These are the default owners for everything in the repo. They will # be requested for review when someone opens a pull request. * @alainfrisch @Drup @pmetzger sedlex-3.8.1/.github/workflows/000077500000000000000000000000001526113672700164165ustar00rootroot00000000000000sedlex-3.8.1/.github/workflows/build.yml000066400000000000000000000023671526113672700202500ustar00rootroot00000000000000name: build on: merge_group: pull_request: push: branches: - master schedule: # Prime the caches every Monday - cron: 0 1 * * MON concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: strategy: fail-fast: false matrix: os: - ubuntu-latest - macos-latest - windows-latest ocaml-compiler: - 4.14.x - 5.x runs-on: ${{ matrix.os }} steps: - name: Checkout code uses: actions/checkout@v4 - name: Use OCaml ${{ matrix.ocaml-compiler }} uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: ${{ matrix.ocaml-compiler }} dune-cache: ${{ matrix.os == 'ubuntu-latest' }} - run: opam install . --with-test - run: opam exec -- make build - run: opam exec -- make test - run: opam exec -- git diff --exit-code lint-fmt: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Use OCaml 4.14.x uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: 4.14.x dune-cache: true - name: Lint fmt uses: ocaml/setup-ocaml/lint-fmt@v3 sedlex-3.8.1/.github/workflows/changelog.yml000066400000000000000000000005211526113672700210660ustar00rootroot00000000000000name: Check changelog on: pull_request: branches: - master types: - labeled - opened - reopened - synchronize - unlabeled jobs: check-changelog: name: Check changelog runs-on: ubuntu-latest steps: - name: Check changelog uses: tarides/changelog-check-action@v3 sedlex-3.8.1/.github/workflows/doc.yml000066400000000000000000000011641526113672700177100ustar00rootroot00000000000000name: Doc build on: push: branches: - master jobs: build_doc: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup OCaml uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: 4.14.x - name: Pin locally run: opam pin -y add --no-action . - name: Install locally run: opam install -y --with-doc sedlex - name: Build doc run: opam exec dune build @doc - name: Deploy doc uses: JamesIves/github-pages-deploy-action@4.1.4 with: branch: gh-pages folder: _build/default/_doc/_html sedlex-3.8.1/.gitignore000066400000000000000000000003241526113672700150100ustar00rootroot00000000000000*.annot *.cmo *.cma *.cmi *.a *.o *.cmx *.cmxs *.cmxa *.exe *.install *.merlin .*sw* _build libdoc src/syntax/ppx_sedlex src/generator/data/*.txt examples/complement examples/tokenizer examples/subtraction _opam sedlex-3.8.1/.merlin000066400000000000000000000001051526113672700143040ustar00rootroot00000000000000PKG compiler-libs ppx_tools ppx_tools.metaquot SRC src/** B src/** sedlex-3.8.1/.ocamlformat000066400000000000000000000003371526113672700153310ustar00rootroot00000000000000version=0.28.1 profile = conventional break-separators = after space-around-lists = false doc-comments = before match-indent = 2 match-indent-nested = always parens-ite exp-grouping = preserve module-item-spacing = compact sedlex-3.8.1/.travis.yml000066400000000000000000000006571526113672700151420ustar00rootroot00000000000000language: c sudo: required install: test -e .travis.opam.sh || wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-opam.sh script: bash -ex .travis-opam.sh env: global: - PACKAGE=sedlex matrix: - OCAML_VERSION=4.04 - OCAML_VERSION=4.05 - OCAML_VERSION=4.06 - OCAML_VERSION=4.07 - OCAML_VERSION=4.08 - OCAML_VERSION=4.09 - OCAML_VERSION=4.10 - OCAML_VERSION=4.11 os: - linux - osx sedlex-3.8.1/CHANGES.md000066400000000000000000000144501526113672700144170ustar00rootroot00000000000000# 3.8.1 (2026-10-06) ## Bug fixes - End of input counts as one more character read, as it did before 3.8: a rule matching `s, eof` beats an earlier rule matching `s`. 3.8 let the earlier rule win, so a lexer with a rule matching the empty string before its `eof` rule looped forever at end of input (#223) # 3.8 (2026-10-01) ## New features - Named capture groups: `(r as x)` binds `x` in the action to a `Sedlexing.submatch`, extracted with `Sedlexing.Utf8.of_submatch`, `Latin1.of_submatch`, `Utf16.of_submatch` or `lexeme_of_submatch`. Both branches of an or-pattern must bind the same names, and `as` is not allowed under `Star`, `Plus`, `Opt`, `Rep`, `Compl`, `Sub` or `Intersect`. Submatches are chosen leftmost-greedy within the longest match, using a tagged DFA; a boundary at a fixed offset from the start or end of the lexeme costs nothing at runtime. A submatch is only valid inside its action (#177, #178, #204, #208, #209, #212) - Nested `let .. in` definitions of `[%sedlex.regexp?]` (#176) - Unicode 18.0.0 (#205) - `Compl`, `Sub` and `Intersect` accept `Rep (c, 1 .. 1)` (#201) - Clearer errors for a bare `Star`, `Plus`, `Opt`, `Utf8`, `Latin1` or `Ascii` and for unknown operators (#196) ## Bug fixes - Rule priority with `eof`: a rule matching `s, eof` no longer beats an earlier rule matching `s` (#211) - End of input is read once: `Plus eof` and `Star ('a' | eof)` no longer loop forever at end of input, and `eof, eof` no longer matches (#211) - Empty character sets (`Chars ""`, `Compl any`, `Sub` or `Intersect` of coinciding classes) are a compile error instead of crashing the ppx or producing a rule that never matches (#201) - Rule sets where every rule is nullable, such as a lone `""`, no longer generate invalid code (#201) - UTF-8 decoding: the surrogate range rejected by `Utf8.Helper.check_three` ended at U+DF00 instead of U+DFFF (#184) - `Sedlexing.sub_lexeme` and its `Latin1`, `Utf8` and `Utf16` versions raise `Invalid_argument` when the range is not within the lexeme, instead of returning code points of the rest of the buffer or, for a negative length, an empty string (#219) ## Performance - Faster `lexeme`, `sub_lexeme` and `of_submatch` in `Sedlexing.Latin1`, `Sedlexing.Utf8` and `Sedlexing.Utf16` (#219) - The memory cells of `as` bindings are no longer cleared at the start of every token (#220) ## Other changes - Require OCaml 4.11, the oldest compiler still in opam-repository # 3.7 (2025-10-06) - Update to unicode 17.0.0 # 3.6 (2025-01-05) - Fixed one of the ranges implementing Implement Corrigendum #1: UTF-8 Shortest Form for 4-bytes long characters (#171) # 3.5 (2025-05-29) - Implement Corrigendum #1: UTF-8 Shortest Form - Add utf8 support for string literal (#127) # 3.4 (2025-03-28) - Make the library compatibility with ppxlib.0.36 (#166) # 3.3 (2024-10-29) - Add support for unicode `16.0.0` (#157) - Add API for retrieving start and stop positions separately (#155) # 3.2 (2023-06-28): - Restore compatibility with OCaml 4.08 - Use `Sedlexing.{Utf8,Utf16}.from_gen` to initialize UTF8 (resp. UTF16) lexing buffers from string. - Delay raising Malformed until actually reading the malformed part of the imput. (#140) - Count lines in all cases (#130). Previously, certain functions for initiating the lexical buffer would disable lines counting. - Check and fix invariants from Cset. The codebase was not respecting invariants documented in the Cset module which could break code relying on it. The code generated by sedlex.ppx could be affected. - Do not rely on comments from unicode UCD files - Add API to track position in bytes. Should be opt-in and backward compatible. (#146) # 3.1: - Fix directly nested sedlex matches (@smuenzel, PR #117, fixes: #12) - Use explicit stdlib in generated code (@hhugo, PR #122, fixes: #115) - Preserve location of lexbuf (@hhugo, PR #118, fixes: #19) - Don't use gen to consume channels (@hhugo, PR #124, fixes: #45) - New expect_test testsuite (@hhugo, PR #124) - Properly recognize malformed truncated input (@hhugo, PR #124) - Raise `Malformed` instead of `Invalid_arg` (@hhugo, PR #126, fixes: #91) - Updated unicode support to `15.0.0` # 3.0: - Dropped `Stream` api which was removed in `4.14.0` ahead of the `5.0` release. 2.6: - Adapted to ppxlib `0.26`, thanks to @pitag-ha 2.5: - Fix exponential compilation time, thanks to @mnxn for reporting in #97 and @fangyi-zhou for fixing in #106 - Update unicode support for `14.0.0`. # 2.4 - Update `dune` support to `2.8`, add auto-generated `opam` files. - Optimize generated code, thanks to @bobzhang - Update unicode version to 13.0.0 # 2.3 - Switch to ppxlib # 2.2 - Support for OCaml 4.08 # 2.1 - GPR#78: Auto-generate unicode data # 2.0 - GPR#70: Switch to dune, opam v2 - GPR#60: Breaking change: switch from int codepoints to Uchar.t codepoints - GPR#59: Track lexing position # 1.99.4 - GPR#47: Switch to ocaml-migrate-parsetree (contributed by Adrien Guatto) - GPR#42: Added 'Rep' (repeat operator) (contributed by jpathy) # 1.99.3 - Update to work with 4.03 (4.02 still supported) # 1.99.2 - First official release of sedlex # 1.99.1 - Support for new Ast_mapper registration API, follow OCaml trunk after the inclusion of the extension_point branch # 1.99 - First version of sedlex. The history below refers to ulex, the ancestor or sedlex implemented with Camlp4. # 1.1 - Generate (more) globally unique identifiers to avoid conflicts when open'ing another module processed by ulex (issue reported by Gerd Stolpmann) # 1.0 - Update to the new Camlp4 and to ocamlbuild (release for OCaml 3.10 only), by Nicolas Pouillard. # 0.8 - Really make it work with OCaml 3.09. - Support for Utf-16. # 0.7 released May 24 2005 - Bug fixes - Update to OCaml 3.09 (currently CVS). Still works with OCaml 3.08. - MIT-like license (used to LGPL) # 0.5 release Jul. 8 2004 - Document how to use a custom implementation for lex buffers - Update to OCaml 3.08 # 0.4 released Jan. 10 2004 - Bug fix (accept 1114111 as valid Unicode code point) - Add the rollback function- # 0.3 released Oct. 8 2003 - Bug fix - Add a new predefined class for ISO identifiers # 0.2 released Sep. 22 2003 - Changed the names of predefined regexp - Fix max_code = 0x10ffff - Lexers that changes encoding on the fly - Documentation of the interface Ulexing # 0.1 released Sep. 20 2003 - Initial release sedlex-3.8.1/HACKING.md000066400000000000000000000015761526113672700144200ustar00rootroot00000000000000# Developer tools ## Visualising DFA graphs in Emacs The expect tests in `test/codegen/` include inline DOT graphs for each lexer. The file `emacs/sedlex-dot.el` lets you render them as SVG images directly inside the test buffer. Load it once: ```elisp (load "/path/to/sedlex/emacs/sedlex-dot.el") ``` Then, in any test buffer containing `DOT:` / `CODE:` blocks: - `M-x sedlex-render-dot-blocks` — replace DOT source with rendered SVG. - `M-x sedlex-remove-dot-images` — remove the overlays and show the source again. Requires `graphviz` (`dot`) in your PATH and Emacs compiled with SVG support (check with `(image-type-available-p 'svg)`). ## Rendering DOT graphs to files `test/codegen/render_dots.sh` extracts all DOT blocks from the expect tests and renders them as SVG files: ``` ./test/codegen/render_dots.sh [output_dir] ``` Output defaults to `test/codegen/dot_images/`. sedlex-3.8.1/LICENSE000066400000000000000000000021101526113672700140200ustar00rootroot00000000000000The MIT License (MIT) Copyright 2005, 2014 by Alain Frisch and LexiFi. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. sedlex-3.8.1/Makefile000066400000000000000000000007231526113672700144630ustar00rootroot00000000000000# The package sedlex is released under the terms of an MIT-like license. # See the attached LICENSE file. # Copyright 2005, 2013 by Alain Frisch and LexiFi. INSTALL_ARGS := $(if $(PREFIX),--prefix $(PREFIX),) .PHONY: build install uninstall clean doc test all build: dune build @install install: dune install $(INSTALL_ARGS) uninstall: dune uninstall $(INSTALL_ARGS) clean: dune clean doc: dune build @doc test: dune build @runtest all: build test doc sedlex-3.8.1/README.md000066400000000000000000000331501526113672700143020ustar00rootroot00000000000000# sedlex [![build](https://github.com/ocaml-community/sedlex/actions/workflows/build.yml/badge.svg)](https://github.com/ocaml-community/sedlex/actions/workflows/build.yml) Unicode-friendly lexer generator for OCaml. This package is licensed by LexiFi under the terms of the MIT license. sedlex was originally written by Alain Frisch and is now maintained as part of the ocaml-community repositories on github. ## API The API is documented [here](https://ocaml-community.github.io/sedlex). ## Overview sedlex is a lexer generator for OCaml, similar to ocamllex, but supporting Unicode. Contrary to ocamllex, lexer specifications for sedlex are embedded in regular OCaml source files. The lexers work with a new kind of "lexbuf", similar to ocamllex Lexing lexbufs, but designed to support Unicode, and abstracting from a specific encoding. A single lexer can work with arbitrary encodings of the input stream. sedlex is the successor of the ulex project. Contrary to ulex which was implemented as a Camlp4 syntax extension, sedlex is based on the new "-ppx" technology of OCaml, which allow rewriting OCaml parse trees through external rewriters. (And what a better name than "sed" for a rewriter?) As any -ppx rewriter, sedlex does not touch the concrete syntax of the language: lexer specifications are written in source file which comply with the standard grammar of OCaml programs. sedlex reuse the syntax for pattern matching in order to describe lexers (regular expressions are encoded within OCaml patterns). A nice consequence is that your editor (vi, emacs, ...) won't get confused (indentation, coloring) and you don't need to learn new priority rules. Moreover, sedlex is compatible with any front-end parsing technology: it works fine even if you use camlp4 or camlp5, with the standard or revised syntax. ## Lexer specifications ### Match rules sedlex adds a new kind of expression to OCaml: lexer definitions. The syntax for the new construction is: ```ocaml match%sedlex lexbuf with | R1 -> e1 ... | Rn -> en | _ -> def ``` or: ```ocaml [%sedlex match lexbuf with | R1 -> e1 ... | Rn -> en | _ -> def ] ``` (The first vertical bar is optional as in any OCaml pattern matching. Guard expressions are not allowed.) where: - lexbuf is an arbitrary lowercase identifier, which must refer to an existing value of type `Sedlexing.lexbuf`. - the Ri are regular expressions (see below); - the ei and def are OCaml expressions (called actions) of the same type (the type for the whole lexer definition). Unlike ocamllex, lexers work on stream of Unicode codepoints, not bytes. Like ocamllex, sedlex uses **longest match** with **first rule priority**: - The lexer always tries to match the longest possible prefix of the input. It does so by continuing to read characters as long as some rule can still match a longer string, while remembering the last position at which a rule did match. - When two or more rules match the same longest prefix (a tie), the rule that appears first in the `match%sedlex` definition wins. For example, given the rules `| "if" -> ...` and `| Plus ('a'..'z') -> ...`, the input `"if"` is matched by the first rule because it is listed first, even though the second rule also accepts `"if"`. - The end of input, matched by `eof`, counts as one more character read: at the end of the input, a rule matching `s, eof` is a longer match than a rule matching `s`, whichever is listed first. In particular, `eof` wins over an earlier rule that matches the empty string. **Important:** The `_` (catch-all) case is *not* a regexp — it is the fallback when no rule matches. Because no characters were consumed, the matched lexeme is empty (`""`). If you need to consume and report an unexpected character, use `any` instead: ```ocaml | any -> Printf.sprintf "unexpected character: %s" (Sedlexing.Utf8.lexeme buf) | _ -> failwith "unreachable: any matches all characters" ``` The actions can call functions from the Sedlexing module to extract (parts of) the matched lexeme, in the desired encoding. ### Regexp syntax Regular expressions are syntactically OCaml patterns: - `"...."` (string constant): recognize the specified string. - `'....'` (character constant): recognize the specified character. - `i` (integer constant): recognize the specified codepoint. - `'...' .. '...'`: character range. - `i1 .. i2`: range between two codepoints. - `R1 | R2`: alternation. - `R, R2, ..., Rn`: concatenation. - `Star R`: Kleene star (0 or more repetition). - `Plus R`: equivalent to `R, R*`. - `Opt R`: equivalent to `("" | R)`. - `Rep (R, n)`: equivalent to `R{n}`. - `Rep (R, n .. m)`: equivalent to `R{n, m}`. - `Chars "..."`: recognize any character in the string. - `Compl R`: assume that R is a single-character length regexp (see below) and recognize the complement set. - `Sub (R1,R2)`: assume that R is a single-character length regexp (see below) and recognize the set of items in `R1` but not in `R2` ("subtract"). - `Intersect (R1,R2)`: assume that `R` is a single-character length regexp (see below) and recognize the set of items which are in both `R1` and `R2`. - `Utf8 R`: string literals inside R are assumed to be utf-8 encoded. - `Latin1 R`: string literals inside R are assumed to be latin1 encoded. - `Ascii R`: string literals inside R are assumed to be ascii encoded. - `lid` (lowercase identifier): reference a named regexp (see below). A single-character length regexp is a regexp which does not contain (after expansion of references) concatenation, Star, Plus, Opt or string constants with a length different from one. ### Precedence Since sedlex regular expressions are encoded as OCaml patterns, they follow OCaml's pattern precedence rules. From lowest to highest: | Precedence | Operator | Example | Parses as | |---|---|---|---| | lowest | `\|` (alternation) | `'a' \| 'b', 'c'` | `'a' \| ('b', 'c')` | | | `,` (concatenation) | `"ab", Star 'c'` | `"ab", (Star 'c')` | | highest | Constructor application (`Star`, `Plus`, `Opt`, ...) | `Star 'a' \| 'b'` | `(Star 'a') \| 'b'` | In particular, `Star r1, r2` is `(Star r1), r2` (not `Star (r1, r2)`), and `r1 | r2, r3` is `r1 | (r2, r3)` (not `(r1 | r2), r3`). Use parentheses to override: `Star (r1, r2)`, `(r1 | r2), r3`. ### Named capture groups (`as` bindings) You can capture sub-matches using OCaml's `as` pattern syntax: ```ocaml match%sedlex buf with | (Plus ('0'..'9') as num), '.', (Plus ('0'..'9') as frac) -> let n = Sedlexing.Utf8.of_submatch num in let f = Sedlexing.Utf8.of_submatch frac in Printf.printf "integer=%s fractional=%s\n" n f | _ -> () ``` Each `as` binding produces a value of type `Sedlexing.submatch`. Use the extraction functions to obtain the matched content: - `Sedlexing.Utf8.of_submatch s` returns the sub-match as a UTF-8 string. - `Sedlexing.Latin1.of_submatch s` returns the sub-match as a Latin-1 string. - `Sedlexing.Utf16.of_submatch s bo bom` returns the sub-match as a UTF-16 string. - `Sedlexing.lexeme_of_submatch s` returns the sub-match as a `Uchar.t array`. A submatch is a view into the lexbuf, valid only until the lexbuf moves on. Extracting it later returns another token's content or raises `Invalid_argument`. Extract inside the action and keep the result: ```ocaml let token buf = match%sedlex buf with | (Plus ('a'..'z') as key), ':' -> KEY (Sedlexing.Utf8.of_submatch key) | _ -> ... ``` Or-patterns work as expected — both sides must bind the same names: ```ocaml match%sedlex buf with | ("0x", Plus hex_digit as n) | (Plus ('0'..'9') as n) -> Sedlexing.Utf8.of_submatch n | _ -> ... ``` When a lexeme can be split several ways, sub-matches are chosen leftmost-greedy within the longest match: the left alternative of `|` wins, and `Star`, `Plus`, `Opt` and `Rep` take as much as they can. In `(Star 'a' as x), Star 'a'`, `x` captures every `a`. **Restriction:** `as` bindings are not allowed inside repetition operators (`Star`, `Plus`, `Opt`, `Rep`), set operators (`Compl`, `Sub`, `Intersect`) or `[%sedlex.regexp? ...]` definitions. ### Encoding - The OCaml source is assumed to be encoded in UTF-8. - Strings and chars literals will be interpreted in ASCII unless otherwise specified by the `Latin1`,`Ascii` and `Utf8` constructors in patterns. ### Named regexps ### Named regular expressions You can give names to regular expressions with `[%sedlex.regexp? ...]` and reference them by name in lexer rules. **Top-level definitions** are visible for the rest of the module: ```ocaml let digit = [%sedlex.regexp? '0' .. '9'] let number = [%sedlex.regexp? Plus digit] let rec token buf = match%sedlex buf with | number -> INT (Sedlexing.Utf8.lexeme buf) | _ -> ... ``` **Local definitions** with `let ... in` are scoped to the body expression: ```ocaml let hex_digit = let digit = [%sedlex.regexp? '0' .. '9'] in let hex_letter = [%sedlex.regexp? 'a' .. 'f' | 'A' .. 'F'] in [%sedlex.regexp? digit | hex_letter] ``` Local definitions also work inside expressions: ```ocaml let token buf = let int_lit = let digit = [%sedlex.regexp? '0' .. '9'] in [%sedlex.regexp? Plus digit] in match%sedlex buf with | int_lit -> ... | _ -> ... ``` ## Predefined regexps sedlex provides a set of predefined regexps: - `any`: any character. - `eof`: the virtual end-of-file character. - `xml_letter`, `xml_digit`, `xml_extender`, `xml_base_char`, `xml_ideographic`, `xml_combining_char`, `xml_blank`: as defined by the XML recommendation. - `tr8876_ident_char`: character names in identifiers from ISO TR8876. - `cc`, `cf`, `cn`, `co`, `cs`, `ll`, `lm`, `lo`, `lt`, `lu`, `mc`, `me`, `mn`, `nd`, `nl`, `no`, `pc`, `pd`, `pe`, `pf`, `pi`, `po`, `ps`, `sc`, `sk`, `sm`, `so`, `zl`, `zp`, `zs`: as defined by the Unicode standard (categories). - `alphabetic`, `ascii_hex_digit`, `hex_digit`, `id_continue`, `id_start`, `lowercase`, `math`, `other_alphabetic`, `other_lowercase`, `other_math`, `other_uppercase`, `uppercase`, `white_space`, `xid_continue`, `xid_start`: as defined by the Unicode standard (properties). ## Running a lexer See the interface of the `Sedlexing` module for a description of how to create lexbuf values (from strings, stream or channels encoded in Latin1, utf8 or utf16, or from integer arrays or streams representing Unicode code points). It is possible to work with a custom implementation for lex buffers. To do this, you just have to ensure that a module called `Sedlexing` is in scope of your lexer specifications, and that it defines at least the following functions: `start`, `next`, `mark`, `backtrack`. See the interface of the `Sedlexing` module for more information. ## Using sedlex The quick way: ``` opam install sedlex ``` Otherwise, the first thing to do is to compile and install sedlex. You need a recent version of OCaml and [dune](https://dune.build/). ``` make ``` ### With findlib If you have findlib, you can use it to install and use sedlex. The name of the findlib package is "sedlex". Installation (after "make"): ``` make install ``` Compilation of OCaml files with lexer specifications: ``` ocamlfind ocamlc -c -package sedlex.ppx my_file.ml ``` When linking, you must also include the sedlex package: ``` ocamlfind ocamlc -o my_prog -linkpkg -package sedlex.ppx my_file.cmo ``` There is also a sedlex.ppx subpackage containing the code of the ppx filter. This can be used to build custom drivers (combining several ppx transformations in a single process). ### Without findlib You can use sedlex without findlib. To compile, you need to run the source file through -ppx rewriter ppx_sedlex. Moreover, you need to link the application with the runtime support library for sedlex (sedlexing.cma / sedlexing.cmxa). ### With utop Once sedlex is installed as per above, simply type ``` #require "sedlex.ppx";; ``` ## Integration with ocamlyacc and menhir sedlex uses its own `Sedlexing.lexbuf` type, while ocamlyacc and menhir (classic API) expect a lexer function of type `Lexing.lexbuf -> token`. To bridge the two, create a dummy `Lexing.lexbuf` and update its position fields after each token: ```ocaml (* In lexer.ml — the sedlex lexer *) let rec token buf = match%sedlex buf with | Plus ('0'..'9') -> Parser.INT (int_of_string (Sedlexing.Utf8.lexeme buf)) | '+' -> Parser.PLUS | Plus white_space -> token buf | eof -> Parser.EOF | _ -> failwith "Unexpected character" (* Wrap for ocamlyacc / menhir classic API *) let tokenize buf = let lexbuf = Lexing.from_string "" in let tokenize lexbuf = let tok = token buf in let start_pos, end_pos = Sedlexing.lexing_positions buf in lexbuf.Lexing.lex_start_p <- start_pos; lexbuf.Lexing.lex_curr_p <- end_pos; tok in (tokenize, lexbuf) (* In main.ml *) let () = let buf = Sedlexing.Utf8.from_string "1 + 2" in let tokenize, lexbuf = Lexer.tokenize buf in let result = Parser.main tokenize lexbuf in ... ``` For menhir's **incremental API**, use `Sedlexing.with_tokenizer` which returns a `unit -> token * position * position` supplier directly: ```ocaml let supplier = Sedlexing.with_tokenizer token buf in let result = Parser.MenhirInterpreter.loop supplier (Parser.Incremental.main Lexing.dummy_pos) in ... ``` Complete working examples are in `examples/with_ocamlyacc/` and `examples/with_menhir/`. ## Examples The `examples/` subdirectory contains several samples of sedlex in use. ## Contributors - Benus Becker: implementation of Utf16 - sghost: for Unicode 6.3 categories and properties - Peter Zotov: - improvements to the build system - switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot) - Steffen Smolka: port to dune - Romain Beauxis: - Implementation of the unicode table extractors - General maintenance sedlex-3.8.1/dune-project000066400000000000000000000017311526113672700153450ustar00rootroot00000000000000(lang dune 3.18) (using menhir 2.1) (version 3.8.1) (name sedlex) (source (github ocaml-community/sedlex)) (license MIT) (authors "Alain Frisch " "https://github.com/ocaml-community/sedlex/graphs/contributors") (maintainers "Alain Frisch ") (homepage "https://github.com/ocaml-community/sedlex") (maintenance_intent "(latest)") (documentation "https://ocaml-community.github.io/sedlex/index.html") (generate_opam_files true) (executables_implicit_empty_intf true) (package (name sedlex) (synopsis "An OCaml lexer generator for Unicode") (description "sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular OCaml source files. Lexing specific constructs are provided via a ppx syntax extension.") (depends (ocaml (>= 4.11)) dune (ppxlib (>= 0.26.0)) gen (ppx_expect :with-test) (menhir :with-test))) sedlex-3.8.1/emacs/000077500000000000000000000000001526113672700141115ustar00rootroot00000000000000sedlex-3.8.1/emacs/sedlex-dot.el000066400000000000000000000030771526113672700165120ustar00rootroot00000000000000;;; sedlex-dot.el --- Render inline DOT graphs in sedlex expect tests -*- lexical-binding: t; -*- (defun sedlex-render-dot-blocks () "Render all DOT blocks in the current buffer as inline SVG images." (interactive) (save-excursion (goto-char (point-min)) (let ((count 0)) (while (re-search-forward "^\\s-*DOT:" nil t) (let ((dot-start (line-beginning-position 2)) (dot-end (if (re-search-forward "^\\s-*CODE:" nil t) (line-beginning-position) nil))) (when dot-end (let* ((dot-src (buffer-substring-no-properties dot-start dot-end)) (svg (with-temp-buffer (insert dot-src) (when (zerop (call-process-region (point-min) (point-max) "dot" t t nil "-Tsvg")) (buffer-string))))) (when svg (goto-char dot-end) (let ((ov (make-overlay dot-start dot-end))) (overlay-put ov 'display (create-image svg 'svg t)) (overlay-put ov 'after-string "\n") (overlay-put ov 'sedlex-dot t) (setq count (1+ count)))))))) (message "Rendered %d DOT graph(s)" count)))) (defun sedlex-remove-dot-images () "Remove all rendered DOT overlays." (interactive) (remove-overlays (point-min) (point-max) 'sedlex-dot t) (message "DOT overlays removed")) (provide 'sedlex-dot) ;;; sedlex-dot.el ends here sedlex-3.8.1/examples/000077500000000000000000000000001526113672700146375ustar00rootroot00000000000000sedlex-3.8.1/examples/complement.ml000066400000000000000000000007051526113672700173360ustar00rootroot00000000000000let ucase = [%sedlex.regexp? 'A' .. 'Z'] let lcase = [%sedlex.regexp? 'a' .. 'z'] let rec token buf = match%sedlex buf with | lcase -> print_char 'L'; token buf | Compl (ucase | lcase) -> print_char '?'; token buf | ucase -> print_char 'U'; token buf | eof -> print_endline "." | _ -> assert false let () = let lexbuf = Sedlexing.Latin1.from_string "Abc::DefG" in token lexbuf sedlex-3.8.1/examples/dune000066400000000000000000000011711526113672700155150ustar00rootroot00000000000000(executables (names tokenizer regressions complement subtraction repeat performance) (libraries sedlex sedlex_ppx sedlex.compiler) (preprocess (pps sedlex.ppx))) (rule (alias runtest) (deps (:< tokenizer.exe)) (action (run %{<}))) (rule (alias runtest) (deps (:< regressions.exe)) (action (run %{<}))) (rule (alias runtest) (deps (:< complement.exe)) (action (run %{<}))) (rule (alias runtest) (deps (:< subtraction.exe)) (action (run %{<}))) (rule (alias runtest) (deps (:< repeat.exe)) (action (run %{<}))) (rule (alias runtest) (deps (:< performance.exe)) (action (run %{<}))) sedlex-3.8.1/examples/performance.ml000066400000000000000000000011051526113672700174670ustar00rootroot00000000000000let rec token buf = match%sedlex buf with any -> token buf | eof -> () | _ -> assert false let time f x = let rec acc f x = function | 0 -> f x | n -> f x |> ignore; acc f x (n - 1) in let t = Sys.time () in let fx = acc f x 10 in Printf.printf "Execution time: %fs\n" (Sys.time () -. t); fx let () = let long_str = String.make 1000000 '\n' in let token_from _ = let lexbuf = Sedlexing.Latin1.from_string long_str in (* let () = Sedlexing.set_curr_p lexbuf Lexing.dummy_pos in *) token lexbuf in time token_from long_str sedlex-3.8.1/examples/regressions.ml000066400000000000000000000040371526113672700175400ustar00rootroot00000000000000(* This test that unicode_old.ml is a strict sub-set of new unicode.ml. *) module CSet = Sedlex_compiler.Cset module Unicode = Sedlex_ppx.Unicode let test_versions = ("17.0.0", "18.0.0") let regressions = [ (* Example *) (* ("lt", CSet.union (CSet.singleton 0x1c5) (CSet.singleton (0x0001))) *) ] let compare name (old_ : CSet.t) (new_ : CSet.t) = let diff = CSet.difference old_ new_ in let regressions = match List.assoc name regressions with | exception Not_found -> CSet.empty | x -> x in (* A regression is a code point of the old set that is expected to be * missing in the new set. *) let regressions_not_in_old = CSet.difference regressions old_ in let regressions_still_in_new = CSet.intersection regressions new_ in let diff = CSet.difference diff regressions in Seq.iter (fun x -> Printf.printf "Invalid regression for 0x%x in %s: absent in old set.\n" x name) (CSet.to_seq regressions_not_in_old); Seq.iter (fun x -> Printf.printf "Invalid regression for 0x%x in %s: still present in new set.\n" x name) (CSet.to_seq regressions_still_in_new); Seq.iter (fun x -> Printf.printf "Code point 0x%x missing in %s!\n" x name) (CSet.to_seq diff) let test new_l (name, old_l) = (* Cn is for unassigned code points, which are allowed to be * used in future version. *) let old_l = Sedlex_compiler.Cset.to_list old_l in if name <> "cn" then ( let old_l = List.fold_left (fun acc (a, b) -> CSet.union acc (CSet.interval a b)) CSet.empty old_l in compare name old_l (List.assoc name new_l)) let () = if (Unicode_old.version, Unicode.version) <> test_versions then failwith (Printf.sprintf "Test written for versions: %s => %s\n%!" Unicode_old.version Unicode.version); Printf.printf "Testing Unicode regression: %s => %s\n%!" Unicode_old.version Unicode.version; List.iter (test Unicode.Categories.list) Unicode_old.Categories.list; List.iter (test Unicode.Properties.list) Unicode_old.Properties.list sedlex-3.8.1/examples/repeat.ml000066400000000000000000000013201526113672700164450ustar00rootroot00000000000000let rec token buf = match%sedlex buf with | white_space -> print_endline "\tWhitespace"; token buf | 'a', Rep (white_space, 1) -> print_endline "a\n\tWhitespace"; token buf | Rep ("bc", 2) -> print_endline "bcbc"; token buf | Rep ("d", 1 .. 1) -> print_endline "d"; token buf | Rep ("ef", 1 .. 3) -> Printf.printf "%s\n" (Sedlexing.Utf8.lexeme buf); token buf | eof -> print_endline "\tEnd" | any -> print_endline "Other"; token buf | _ -> failwith "Internal failure: Reached impossible place" let () = let lexbuf = Sedlexing.Utf8.from_string "a bcbc d ef efef efefef" in token lexbuf sedlex-3.8.1/examples/subtraction.ml000066400000000000000000000011251526113672700175250ustar00rootroot00000000000000let rec token buf = match%sedlex buf with | white_space -> print_endline "\tWhitespace"; token buf | Sub (Chars "ab", "b") -> print_endline "a"; token buf | Chars "ab" | "c" -> print_endline "abc"; token buf | Intersect ("d", Chars "abd") -> print_endline "d"; token buf | eof -> print_endline "\tEnd" | any -> print_endline "Other"; token buf | _ -> failwith "Internal failure: Reached impossible place" let () = let lexbuf = Sedlexing.Utf8.from_string "a b c d e" in token lexbuf sedlex-3.8.1/examples/tokenizer.ml000066400000000000000000000014401526113672700172020ustar00rootroot00000000000000let digit = [%sedlex.regexp? '0' .. '9'] let number = [%sedlex.regexp? Plus digit] let rec token buf = let letter = [%sedlex.regexp? 'a' .. 'z' | 'A' .. 'Z'] in match%sedlex buf with | number -> Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); token buf | letter, Star ('A' .. 'Z' | 'a' .. 'z' | digit) -> Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf); token buf | Plus xml_blank -> token buf | Plus (Chars "+*-/") -> Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf); token buf | 128 .. 255 -> print_endline "Non ASCII" | eof -> print_endline "EOF" | _ -> failwith "Unexpected character" let () = let lexbuf = Sedlexing.Latin1.from_string "foobar A123Bfoo ++123Xbar/foo" in token lexbuf sedlex-3.8.1/examples/unicode_old.ml000066400000000000000000005157141526113672700174720ustar00rootroot00000000000000[@@@ocamlformat "disable"] (* This file was automatically generated, do not edit. *) (* Edit gen_unicode.ml.inc instead. *) open Sedlex_compiler let version = "17.0.0" module Categories = struct let cc = Cset.of_list [0x0, 0x1f; 0x7f, 0x9f] let cf = Cset.of_list [0xad, 0xad; 0x600, 0x605; 0x61c, 0x61c; 0x6dd, 0x6dd; 0x70f, 0x70f; 0x890, 0x891; 0x8e2, 0x8e2; 0x180e, 0x180e; 0x200b, 0x200f; 0x202a, 0x202e; 0x2060, 0x2064; 0x2066, 0x206f; 0xfeff, 0xfeff; 0xfff9, 0xfffb; 0x110bd, 0x110bd; 0x110cd, 0x110cd; 0x13430, 0x1343f; 0x1bca0, 0x1bca3; 0x1d173, 0x1d17a; 0xe0001, 0xe0001; 0xe0020, 0xe007f] let cn = Cset.of_list [0x378, 0x379; 0x380, 0x383; 0x38b, 0x38b; 0x38d, 0x38d; 0x3a2, 0x3a2; 0x530, 0x530; 0x557, 0x558; 0x58b, 0x58c; 0x590, 0x590; 0x5c8, 0x5cf; 0x5eb, 0x5ee; 0x5f5, 0x5ff; 0x70e, 0x70e; 0x74b, 0x74c; 0x7b2, 0x7bf; 0x7fb, 0x7fc; 0x82e, 0x82f; 0x83f, 0x83f; 0x85c, 0x85d; 0x85f, 0x85f; 0x86b, 0x86f; 0x892, 0x896; 0x984, 0x984; 0x98d, 0x98e; 0x991, 0x992; 0x9a9, 0x9a9; 0x9b1, 0x9b1; 0x9b3, 0x9b5; 0x9ba, 0x9bb; 0x9c5, 0x9c6; 0x9c9, 0x9ca; 0x9cf, 0x9d6; 0x9d8, 0x9db; 0x9de, 0x9de; 0x9e4, 0x9e5; 0x9ff, 0xa00; 0xa04, 0xa04; 0xa0b, 0xa0e; 0xa11, 0xa12; 0xa29, 0xa29; 0xa31, 0xa31; 0xa34, 0xa34; 0xa37, 0xa37; 0xa3a, 0xa3b; 0xa3d, 0xa3d; 0xa43, 0xa46; 0xa49, 0xa4a; 0xa4e, 0xa50; 0xa52, 0xa58; 0xa5d, 0xa5d; 0xa5f, 0xa65; 0xa77, 0xa80; 0xa84, 0xa84; 0xa8e, 0xa8e; 0xa92, 0xa92; 0xaa9, 0xaa9; 0xab1, 0xab1; 0xab4, 0xab4; 0xaba, 0xabb; 0xac6, 0xac6; 0xaca, 0xaca; 0xace, 0xacf; 0xad1, 0xadf; 0xae4, 0xae5; 0xaf2, 0xaf8; 0xb00, 0xb00; 0xb04, 0xb04; 0xb0d, 0xb0e; 0xb11, 0xb12; 0xb29, 0xb29; 0xb31, 0xb31; 0xb34, 0xb34; 0xb3a, 0xb3b; 0xb45, 0xb46; 0xb49, 0xb4a; 0xb4e, 0xb54; 0xb58, 0xb5b; 0xb5e, 0xb5e; 0xb64, 0xb65; 0xb78, 0xb81; 0xb84, 0xb84; 0xb8b, 0xb8d; 0xb91, 0xb91; 0xb96, 0xb98; 0xb9b, 0xb9b; 0xb9d, 0xb9d; 0xba0, 0xba2; 0xba5, 0xba7; 0xbab, 0xbad; 0xbba, 0xbbd; 0xbc3, 0xbc5; 0xbc9, 0xbc9; 0xbce, 0xbcf; 0xbd1, 0xbd6; 0xbd8, 0xbe5; 0xbfb, 0xbff; 0xc0d, 0xc0d; 0xc11, 0xc11; 0xc29, 0xc29; 0xc3a, 0xc3b; 0xc45, 0xc45; 0xc49, 0xc49; 0xc4e, 0xc54; 0xc57, 0xc57; 0xc5b, 0xc5b; 0xc5e, 0xc5f; 0xc64, 0xc65; 0xc70, 0xc76; 0xc8d, 0xc8d; 0xc91, 0xc91; 0xca9, 0xca9; 0xcb4, 0xcb4; 0xcba, 0xcbb; 0xcc5, 0xcc5; 0xcc9, 0xcc9; 0xcce, 0xcd4; 0xcd7, 0xcdb; 0xcdf, 0xcdf; 0xce4, 0xce5; 0xcf0, 0xcf0; 0xcf4, 0xcff; 0xd0d, 0xd0d; 0xd11, 0xd11; 0xd45, 0xd45; 0xd49, 0xd49; 0xd50, 0xd53; 0xd64, 0xd65; 0xd80, 0xd80; 0xd84, 0xd84; 0xd97, 0xd99; 0xdb2, 0xdb2; 0xdbc, 0xdbc; 0xdbe, 0xdbf; 0xdc7, 0xdc9; 0xdcb, 0xdce; 0xdd5, 0xdd5; 0xdd7, 0xdd7; 0xde0, 0xde5; 0xdf0, 0xdf1; 0xdf5, 0xe00; 0xe3b, 0xe3e; 0xe5c, 0xe80; 0xe83, 0xe83; 0xe85, 0xe85; 0xe8b, 0xe8b; 0xea4, 0xea4; 0xea6, 0xea6; 0xebe, 0xebf; 0xec5, 0xec5; 0xec7, 0xec7; 0xecf, 0xecf; 0xeda, 0xedb; 0xee0, 0xeff; 0xf48, 0xf48; 0xf6d, 0xf70; 0xf98, 0xf98; 0xfbd, 0xfbd; 0xfcd, 0xfcd; 0xfdb, 0xfff; 0x10c6, 0x10c6; 0x10c8, 0x10cc; 0x10ce, 0x10cf; 0x1249, 0x1249; 0x124e, 0x124f; 0x1257, 0x1257; 0x1259, 0x1259; 0x125e, 0x125f; 0x1289, 0x1289; 0x128e, 0x128f; 0x12b1, 0x12b1; 0x12b6, 0x12b7; 0x12bf, 0x12bf; 0x12c1, 0x12c1; 0x12c6, 0x12c7; 0x12d7, 0x12d7; 0x1311, 0x1311; 0x1316, 0x1317; 0x135b, 0x135c; 0x137d, 0x137f; 0x139a, 0x139f; 0x13f6, 0x13f7; 0x13fe, 0x13ff; 0x169d, 0x169f; 0x16f9, 0x16ff; 0x1716, 0x171e; 0x1737, 0x173f; 0x1754, 0x175f; 0x176d, 0x176d; 0x1771, 0x1771; 0x1774, 0x177f; 0x17de, 0x17df; 0x17ea, 0x17ef; 0x17fa, 0x17ff; 0x181a, 0x181f; 0x1879, 0x187f; 0x18ab, 0x18af; 0x18f6, 0x18ff; 0x191f, 0x191f; 0x192c, 0x192f; 0x193c, 0x193f; 0x1941, 0x1943; 0x196e, 0x196f; 0x1975, 0x197f; 0x19ac, 0x19af; 0x19ca, 0x19cf; 0x19db, 0x19dd; 0x1a1c, 0x1a1d; 0x1a5f, 0x1a5f; 0x1a7d, 0x1a7e; 0x1a8a, 0x1a8f; 0x1a9a, 0x1a9f; 0x1aae, 0x1aaf; 0x1ade, 0x1adf; 0x1aec, 0x1aff; 0x1b4d, 0x1b4d; 0x1bf4, 0x1bfb; 0x1c38, 0x1c3a; 0x1c4a, 0x1c4c; 0x1c8b, 0x1c8f; 0x1cbb, 0x1cbc; 0x1cc8, 0x1ccf; 0x1cfb, 0x1cff; 0x1f16, 0x1f17; 0x1f1e, 0x1f1f; 0x1f46, 0x1f47; 0x1f4e, 0x1f4f; 0x1f58, 0x1f58; 0x1f5a, 0x1f5a; 0x1f5c, 0x1f5c; 0x1f5e, 0x1f5e; 0x1f7e, 0x1f7f; 0x1fb5, 0x1fb5; 0x1fc5, 0x1fc5; 0x1fd4, 0x1fd5; 0x1fdc, 0x1fdc; 0x1ff0, 0x1ff1; 0x1ff5, 0x1ff5; 0x1fff, 0x1fff; 0x2065, 0x2065; 0x2072, 0x2073; 0x208f, 0x208f; 0x209d, 0x209f; 0x20c2, 0x20cf; 0x20f1, 0x20ff; 0x218c, 0x218f; 0x242a, 0x243f; 0x244b, 0x245f; 0x2b74, 0x2b75; 0x2cf4, 0x2cf8; 0x2d26, 0x2d26; 0x2d28, 0x2d2c; 0x2d2e, 0x2d2f; 0x2d68, 0x2d6e; 0x2d71, 0x2d7e; 0x2d97, 0x2d9f; 0x2da7, 0x2da7; 0x2daf, 0x2daf; 0x2db7, 0x2db7; 0x2dbf, 0x2dbf; 0x2dc7, 0x2dc7; 0x2dcf, 0x2dcf; 0x2dd7, 0x2dd7; 0x2ddf, 0x2ddf; 0x2e5e, 0x2e7f; 0x2e9a, 0x2e9a; 0x2ef4, 0x2eff; 0x2fd6, 0x2fef; 0x3040, 0x3040; 0x3097, 0x3098; 0x3100, 0x3104; 0x3130, 0x3130; 0x318f, 0x318f; 0x31e6, 0x31ee; 0x321f, 0x321f; 0xa48d, 0xa48f; 0xa4c7, 0xa4cf; 0xa62c, 0xa63f; 0xa6f8, 0xa6ff; 0xa7dd, 0xa7f0; 0xa82d, 0xa82f; 0xa83a, 0xa83f; 0xa878, 0xa87f; 0xa8c6, 0xa8cd; 0xa8da, 0xa8df; 0xa954, 0xa95e; 0xa97d, 0xa97f; 0xa9ce, 0xa9ce; 0xa9da, 0xa9dd; 0xa9ff, 0xa9ff; 0xaa37, 0xaa3f; 0xaa4e, 0xaa4f; 0xaa5a, 0xaa5b; 0xaac3, 0xaada; 0xaaf7, 0xab00; 0xab07, 0xab08; 0xab0f, 0xab10; 0xab17, 0xab1f; 0xab27, 0xab27; 0xab2f, 0xab2f; 0xab6c, 0xab6f; 0xabee, 0xabef; 0xabfa, 0xabff; 0xd7a4, 0xd7af; 0xd7c7, 0xd7ca; 0xd7fc, 0xd7ff; 0xfa6e, 0xfa6f; 0xfada, 0xfaff; 0xfb07, 0xfb12; 0xfb18, 0xfb1c; 0xfb37, 0xfb37; 0xfb3d, 0xfb3d; 0xfb3f, 0xfb3f; 0xfb42, 0xfb42; 0xfb45, 0xfb45; 0xfdd0, 0xfdef; 0xfe1a, 0xfe1f; 0xfe53, 0xfe53; 0xfe67, 0xfe67; 0xfe6c, 0xfe6f; 0xfe75, 0xfe75; 0xfefd, 0xfefe; 0xff00, 0xff00; 0xffbf, 0xffc1; 0xffc8, 0xffc9; 0xffd0, 0xffd1; 0xffd8, 0xffd9; 0xffdd, 0xffdf; 0xffe7, 0xffe7; 0xffef, 0xfff8; 0xfffe, 0xffff; 0x1000c, 0x1000c; 0x10027, 0x10027; 0x1003b, 0x1003b; 0x1003e, 0x1003e; 0x1004e, 0x1004f; 0x1005e, 0x1007f; 0x100fb, 0x100ff; 0x10103, 0x10106; 0x10134, 0x10136; 0x1018f, 0x1018f; 0x1019d, 0x1019f; 0x101a1, 0x101cf; 0x101fe, 0x1027f; 0x1029d, 0x1029f; 0x102d1, 0x102df; 0x102fc, 0x102ff; 0x10324, 0x1032c; 0x1034b, 0x1034f; 0x1037b, 0x1037f; 0x1039e, 0x1039e; 0x103c4, 0x103c7; 0x103d6, 0x103ff; 0x1049e, 0x1049f; 0x104aa, 0x104af; 0x104d4, 0x104d7; 0x104fc, 0x104ff; 0x10528, 0x1052f; 0x10564, 0x1056e; 0x1057b, 0x1057b; 0x1058b, 0x1058b; 0x10593, 0x10593; 0x10596, 0x10596; 0x105a2, 0x105a2; 0x105b2, 0x105b2; 0x105ba, 0x105ba; 0x105bd, 0x105bf; 0x105f4, 0x105ff; 0x10737, 0x1073f; 0x10756, 0x1075f; 0x10768, 0x1077f; 0x10786, 0x10786; 0x107b1, 0x107b1; 0x107bb, 0x107ff; 0x10806, 0x10807; 0x10809, 0x10809; 0x10836, 0x10836; 0x10839, 0x1083b; 0x1083d, 0x1083e; 0x10856, 0x10856; 0x1089f, 0x108a6; 0x108b0, 0x108df; 0x108f3, 0x108f3; 0x108f6, 0x108fa; 0x1091c, 0x1091e; 0x1093a, 0x1093e; 0x1095a, 0x1097f; 0x109b8, 0x109bb; 0x109d0, 0x109d1; 0x10a04, 0x10a04; 0x10a07, 0x10a0b; 0x10a14, 0x10a14; 0x10a18, 0x10a18; 0x10a36, 0x10a37; 0x10a3b, 0x10a3e; 0x10a49, 0x10a4f; 0x10a59, 0x10a5f; 0x10aa0, 0x10abf; 0x10ae7, 0x10aea; 0x10af7, 0x10aff; 0x10b36, 0x10b38; 0x10b56, 0x10b57; 0x10b73, 0x10b77; 0x10b92, 0x10b98; 0x10b9d, 0x10ba8; 0x10bb0, 0x10bff; 0x10c49, 0x10c7f; 0x10cb3, 0x10cbf; 0x10cf3, 0x10cf9; 0x10d28, 0x10d2f; 0x10d3a, 0x10d3f; 0x10d66, 0x10d68; 0x10d86, 0x10d8d; 0x10d90, 0x10e5f; 0x10e7f, 0x10e7f; 0x10eaa, 0x10eaa; 0x10eae, 0x10eaf; 0x10eb2, 0x10ec1; 0x10ec8, 0x10ecf; 0x10ed9, 0x10ef9; 0x10f28, 0x10f2f; 0x10f5a, 0x10f6f; 0x10f8a, 0x10faf; 0x10fcc, 0x10fdf; 0x10ff7, 0x10fff; 0x1104e, 0x11051; 0x11076, 0x1107e; 0x110c3, 0x110cc; 0x110ce, 0x110cf; 0x110e9, 0x110ef; 0x110fa, 0x110ff; 0x11135, 0x11135; 0x11148, 0x1114f; 0x11177, 0x1117f; 0x111e0, 0x111e0; 0x111f5, 0x111ff; 0x11212, 0x11212; 0x11242, 0x1127f; 0x11287, 0x11287; 0x11289, 0x11289; 0x1128e, 0x1128e; 0x1129e, 0x1129e; 0x112aa, 0x112af; 0x112eb, 0x112ef; 0x112fa, 0x112ff; 0x11304, 0x11304; 0x1130d, 0x1130e; 0x11311, 0x11312; 0x11329, 0x11329; 0x11331, 0x11331; 0x11334, 0x11334; 0x1133a, 0x1133a; 0x11345, 0x11346; 0x11349, 0x1134a; 0x1134e, 0x1134f; 0x11351, 0x11356; 0x11358, 0x1135c; 0x11364, 0x11365; 0x1136d, 0x1136f; 0x11375, 0x1137f; 0x1138a, 0x1138a; 0x1138c, 0x1138d; 0x1138f, 0x1138f; 0x113b6, 0x113b6; 0x113c1, 0x113c1; 0x113c3, 0x113c4; 0x113c6, 0x113c6; 0x113cb, 0x113cb; 0x113d6, 0x113d6; 0x113d9, 0x113e0; 0x113e3, 0x113ff; 0x1145c, 0x1145c; 0x11462, 0x1147f; 0x114c8, 0x114cf; 0x114da, 0x1157f; 0x115b6, 0x115b7; 0x115de, 0x115ff; 0x11645, 0x1164f; 0x1165a, 0x1165f; 0x1166d, 0x1167f; 0x116ba, 0x116bf; 0x116ca, 0x116cf; 0x116e4, 0x116ff; 0x1171b, 0x1171c; 0x1172c, 0x1172f; 0x11747, 0x117ff; 0x1183c, 0x1189f; 0x118f3, 0x118fe; 0x11907, 0x11908; 0x1190a, 0x1190b; 0x11914, 0x11914; 0x11917, 0x11917; 0x11936, 0x11936; 0x11939, 0x1193a; 0x11947, 0x1194f; 0x1195a, 0x1199f; 0x119a8, 0x119a9; 0x119d8, 0x119d9; 0x119e5, 0x119ff; 0x11a48, 0x11a4f; 0x11aa3, 0x11aaf; 0x11af9, 0x11aff; 0x11b0a, 0x11b5f; 0x11b68, 0x11bbf; 0x11be2, 0x11bef; 0x11bfa, 0x11bff; 0x11c09, 0x11c09; 0x11c37, 0x11c37; 0x11c46, 0x11c4f; 0x11c6d, 0x11c6f; 0x11c90, 0x11c91; 0x11ca8, 0x11ca8; 0x11cb7, 0x11cff; 0x11d07, 0x11d07; 0x11d0a, 0x11d0a; 0x11d37, 0x11d39; 0x11d3b, 0x11d3b; 0x11d3e, 0x11d3e; 0x11d48, 0x11d4f; 0x11d5a, 0x11d5f; 0x11d66, 0x11d66; 0x11d69, 0x11d69; 0x11d8f, 0x11d8f; 0x11d92, 0x11d92; 0x11d99, 0x11d9f; 0x11daa, 0x11daf; 0x11ddc, 0x11ddf; 0x11dea, 0x11edf; 0x11ef9, 0x11eff; 0x11f11, 0x11f11; 0x11f3b, 0x11f3d; 0x11f5b, 0x11faf; 0x11fb1, 0x11fbf; 0x11ff2, 0x11ffe; 0x1239a, 0x123ff; 0x1246f, 0x1246f; 0x12475, 0x1247f; 0x12544, 0x12f8f; 0x12ff3, 0x12fff; 0x13456, 0x1345f; 0x143fb, 0x143ff; 0x14647, 0x160ff; 0x1613a, 0x167ff; 0x16a39, 0x16a3f; 0x16a5f, 0x16a5f; 0x16a6a, 0x16a6d; 0x16abf, 0x16abf; 0x16aca, 0x16acf; 0x16aee, 0x16aef; 0x16af6, 0x16aff; 0x16b46, 0x16b4f; 0x16b5a, 0x16b5a; 0x16b62, 0x16b62; 0x16b78, 0x16b7c; 0x16b90, 0x16d3f; 0x16d7a, 0x16e3f; 0x16e9b, 0x16e9f; 0x16eb9, 0x16eba; 0x16ed4, 0x16eff; 0x16f4b, 0x16f4e; 0x16f88, 0x16f8e; 0x16fa0, 0x16fdf; 0x16fe5, 0x16fef; 0x16ff7, 0x16fff; 0x18cd6, 0x18cfe; 0x18d1f, 0x18d7f; 0x18df3, 0x1afef; 0x1aff4, 0x1aff4; 0x1affc, 0x1affc; 0x1afff, 0x1afff; 0x1b123, 0x1b131; 0x1b133, 0x1b14f; 0x1b153, 0x1b154; 0x1b156, 0x1b163; 0x1b168, 0x1b16f; 0x1b2fc, 0x1bbff; 0x1bc6b, 0x1bc6f; 0x1bc7d, 0x1bc7f; 0x1bc89, 0x1bc8f; 0x1bc9a, 0x1bc9b; 0x1bca4, 0x1cbff; 0x1ccfd, 0x1ccff; 0x1ceb4, 0x1ceb9; 0x1ced1, 0x1cedf; 0x1cef1, 0x1ceff; 0x1cf2e, 0x1cf2f; 0x1cf47, 0x1cf4f; 0x1cfc4, 0x1cfff; 0x1d0f6, 0x1d0ff; 0x1d127, 0x1d128; 0x1d1eb, 0x1d1ff; 0x1d246, 0x1d2bf; 0x1d2d4, 0x1d2df; 0x1d2f4, 0x1d2ff; 0x1d357, 0x1d35f; 0x1d379, 0x1d3ff; 0x1d455, 0x1d455; 0x1d49d, 0x1d49d; 0x1d4a0, 0x1d4a1; 0x1d4a3, 0x1d4a4; 0x1d4a7, 0x1d4a8; 0x1d4ad, 0x1d4ad; 0x1d4ba, 0x1d4ba; 0x1d4bc, 0x1d4bc; 0x1d4c4, 0x1d4c4; 0x1d506, 0x1d506; 0x1d50b, 0x1d50c; 0x1d515, 0x1d515; 0x1d51d, 0x1d51d; 0x1d53a, 0x1d53a; 0x1d53f, 0x1d53f; 0x1d545, 0x1d545; 0x1d547, 0x1d549; 0x1d551, 0x1d551; 0x1d6a6, 0x1d6a7; 0x1d7cc, 0x1d7cd; 0x1da8c, 0x1da9a; 0x1daa0, 0x1daa0; 0x1dab0, 0x1deff; 0x1df1f, 0x1df24; 0x1df2b, 0x1dfff; 0x1e007, 0x1e007; 0x1e019, 0x1e01a; 0x1e022, 0x1e022; 0x1e025, 0x1e025; 0x1e02b, 0x1e02f; 0x1e06e, 0x1e08e; 0x1e090, 0x1e0ff; 0x1e12d, 0x1e12f; 0x1e13e, 0x1e13f; 0x1e14a, 0x1e14d; 0x1e150, 0x1e28f; 0x1e2af, 0x1e2bf; 0x1e2fa, 0x1e2fe; 0x1e300, 0x1e4cf; 0x1e4fa, 0x1e5cf; 0x1e5fb, 0x1e5fe; 0x1e600, 0x1e6bf; 0x1e6df, 0x1e6df; 0x1e6f6, 0x1e6fd; 0x1e700, 0x1e7df; 0x1e7e7, 0x1e7e7; 0x1e7ec, 0x1e7ec; 0x1e7ef, 0x1e7ef; 0x1e7ff, 0x1e7ff; 0x1e8c5, 0x1e8c6; 0x1e8d7, 0x1e8ff; 0x1e94c, 0x1e94f; 0x1e95a, 0x1e95d; 0x1e960, 0x1ec70; 0x1ecb5, 0x1ed00; 0x1ed3e, 0x1edff; 0x1ee04, 0x1ee04; 0x1ee20, 0x1ee20; 0x1ee23, 0x1ee23; 0x1ee25, 0x1ee26; 0x1ee28, 0x1ee28; 0x1ee33, 0x1ee33; 0x1ee38, 0x1ee38; 0x1ee3a, 0x1ee3a; 0x1ee3c, 0x1ee41; 0x1ee43, 0x1ee46; 0x1ee48, 0x1ee48; 0x1ee4a, 0x1ee4a; 0x1ee4c, 0x1ee4c; 0x1ee50, 0x1ee50; 0x1ee53, 0x1ee53; 0x1ee55, 0x1ee56; 0x1ee58, 0x1ee58; 0x1ee5a, 0x1ee5a; 0x1ee5c, 0x1ee5c; 0x1ee5e, 0x1ee5e; 0x1ee60, 0x1ee60; 0x1ee63, 0x1ee63; 0x1ee65, 0x1ee66; 0x1ee6b, 0x1ee6b; 0x1ee73, 0x1ee73; 0x1ee78, 0x1ee78; 0x1ee7d, 0x1ee7d; 0x1ee7f, 0x1ee7f; 0x1ee8a, 0x1ee8a; 0x1ee9c, 0x1eea0; 0x1eea4, 0x1eea4; 0x1eeaa, 0x1eeaa; 0x1eebc, 0x1eeef; 0x1eef2, 0x1efff; 0x1f02c, 0x1f02f; 0x1f094, 0x1f09f; 0x1f0af, 0x1f0b0; 0x1f0c0, 0x1f0c0; 0x1f0d0, 0x1f0d0; 0x1f0f6, 0x1f0ff; 0x1f1ae, 0x1f1e5; 0x1f203, 0x1f20f; 0x1f23c, 0x1f23f; 0x1f249, 0x1f24f; 0x1f252, 0x1f25f; 0x1f266, 0x1f2ff; 0x1f6d9, 0x1f6db; 0x1f6ed, 0x1f6ef; 0x1f6fd, 0x1f6ff; 0x1f7da, 0x1f7df; 0x1f7ec, 0x1f7ef; 0x1f7f1, 0x1f7ff; 0x1f80c, 0x1f80f; 0x1f848, 0x1f84f; 0x1f85a, 0x1f85f; 0x1f888, 0x1f88f; 0x1f8ae, 0x1f8af; 0x1f8bc, 0x1f8bf; 0x1f8c2, 0x1f8cf; 0x1f8d9, 0x1f8ff; 0x1fa58, 0x1fa5f; 0x1fa6e, 0x1fa6f; 0x1fa7d, 0x1fa7f; 0x1fa8b, 0x1fa8d; 0x1fac7, 0x1fac7; 0x1fac9, 0x1facc; 0x1fadd, 0x1fade; 0x1faeb, 0x1faee; 0x1faf9, 0x1faff; 0x1fb93, 0x1fb93; 0x1fbfb, 0x1ffff; 0x2a6e0, 0x2a6ff; 0x2b81e, 0x2b81f; 0x2ceae, 0x2ceaf; 0x2ebe1, 0x2ebef; 0x2ee5e, 0x2f7ff; 0x2fa1e, 0x2ffff; 0x3134b, 0x3134f; 0x3347a, 0xe0000; 0xe0002, 0xe001f; 0xe0080, 0xe00ff; 0xe01f0, 0xeffff; 0xffffe, 0xfffff; 0x10fffe, 0x10ffff] let co = Cset.of_list [0xe000, 0xf8ff; 0xf0000, 0xffffd; 0x100000, 0x10fffd] let cs = Cset.of_list [0xd800, 0xdfff] let ll = Cset.of_list [0x61, 0x7a; 0xb5, 0xb5; 0xdf, 0xf6; 0xf8, 0xff; 0x101, 0x101; 0x103, 0x103; 0x105, 0x105; 0x107, 0x107; 0x109, 0x109; 0x10b, 0x10b; 0x10d, 0x10d; 0x10f, 0x10f; 0x111, 0x111; 0x113, 0x113; 0x115, 0x115; 0x117, 0x117; 0x119, 0x119; 0x11b, 0x11b; 0x11d, 0x11d; 0x11f, 0x11f; 0x121, 0x121; 0x123, 0x123; 0x125, 0x125; 0x127, 0x127; 0x129, 0x129; 0x12b, 0x12b; 0x12d, 0x12d; 0x12f, 0x12f; 0x131, 0x131; 0x133, 0x133; 0x135, 0x135; 0x137, 0x138; 0x13a, 0x13a; 0x13c, 0x13c; 0x13e, 0x13e; 0x140, 0x140; 0x142, 0x142; 0x144, 0x144; 0x146, 0x146; 0x148, 0x149; 0x14b, 0x14b; 0x14d, 0x14d; 0x14f, 0x14f; 0x151, 0x151; 0x153, 0x153; 0x155, 0x155; 0x157, 0x157; 0x159, 0x159; 0x15b, 0x15b; 0x15d, 0x15d; 0x15f, 0x15f; 0x161, 0x161; 0x163, 0x163; 0x165, 0x165; 0x167, 0x167; 0x169, 0x169; 0x16b, 0x16b; 0x16d, 0x16d; 0x16f, 0x16f; 0x171, 0x171; 0x173, 0x173; 0x175, 0x175; 0x177, 0x177; 0x17a, 0x17a; 0x17c, 0x17c; 0x17e, 0x180; 0x183, 0x183; 0x185, 0x185; 0x188, 0x188; 0x18c, 0x18d; 0x192, 0x192; 0x195, 0x195; 0x199, 0x19b; 0x19e, 0x19e; 0x1a1, 0x1a1; 0x1a3, 0x1a3; 0x1a5, 0x1a5; 0x1a8, 0x1a8; 0x1aa, 0x1ab; 0x1ad, 0x1ad; 0x1b0, 0x1b0; 0x1b4, 0x1b4; 0x1b6, 0x1b6; 0x1b9, 0x1ba; 0x1bd, 0x1bf; 0x1c6, 0x1c6; 0x1c9, 0x1c9; 0x1cc, 0x1cc; 0x1ce, 0x1ce; 0x1d0, 0x1d0; 0x1d2, 0x1d2; 0x1d4, 0x1d4; 0x1d6, 0x1d6; 0x1d8, 0x1d8; 0x1da, 0x1da; 0x1dc, 0x1dd; 0x1df, 0x1df; 0x1e1, 0x1e1; 0x1e3, 0x1e3; 0x1e5, 0x1e5; 0x1e7, 0x1e7; 0x1e9, 0x1e9; 0x1eb, 0x1eb; 0x1ed, 0x1ed; 0x1ef, 0x1f0; 0x1f3, 0x1f3; 0x1f5, 0x1f5; 0x1f9, 0x1f9; 0x1fb, 0x1fb; 0x1fd, 0x1fd; 0x1ff, 0x1ff; 0x201, 0x201; 0x203, 0x203; 0x205, 0x205; 0x207, 0x207; 0x209, 0x209; 0x20b, 0x20b; 0x20d, 0x20d; 0x20f, 0x20f; 0x211, 0x211; 0x213, 0x213; 0x215, 0x215; 0x217, 0x217; 0x219, 0x219; 0x21b, 0x21b; 0x21d, 0x21d; 0x21f, 0x21f; 0x221, 0x221; 0x223, 0x223; 0x225, 0x225; 0x227, 0x227; 0x229, 0x229; 0x22b, 0x22b; 0x22d, 0x22d; 0x22f, 0x22f; 0x231, 0x231; 0x233, 0x239; 0x23c, 0x23c; 0x23f, 0x240; 0x242, 0x242; 0x247, 0x247; 0x249, 0x249; 0x24b, 0x24b; 0x24d, 0x24d; 0x24f, 0x293; 0x296, 0x2af; 0x371, 0x371; 0x373, 0x373; 0x377, 0x377; 0x37b, 0x37d; 0x390, 0x390; 0x3ac, 0x3ce; 0x3d0, 0x3d1; 0x3d5, 0x3d7; 0x3d9, 0x3d9; 0x3db, 0x3db; 0x3dd, 0x3dd; 0x3df, 0x3df; 0x3e1, 0x3e1; 0x3e3, 0x3e3; 0x3e5, 0x3e5; 0x3e7, 0x3e7; 0x3e9, 0x3e9; 0x3eb, 0x3eb; 0x3ed, 0x3ed; 0x3ef, 0x3f3; 0x3f5, 0x3f5; 0x3f8, 0x3f8; 0x3fb, 0x3fc; 0x430, 0x45f; 0x461, 0x461; 0x463, 0x463; 0x465, 0x465; 0x467, 0x467; 0x469, 0x469; 0x46b, 0x46b; 0x46d, 0x46d; 0x46f, 0x46f; 0x471, 0x471; 0x473, 0x473; 0x475, 0x475; 0x477, 0x477; 0x479, 0x479; 0x47b, 0x47b; 0x47d, 0x47d; 0x47f, 0x47f; 0x481, 0x481; 0x48b, 0x48b; 0x48d, 0x48d; 0x48f, 0x48f; 0x491, 0x491; 0x493, 0x493; 0x495, 0x495; 0x497, 0x497; 0x499, 0x499; 0x49b, 0x49b; 0x49d, 0x49d; 0x49f, 0x49f; 0x4a1, 0x4a1; 0x4a3, 0x4a3; 0x4a5, 0x4a5; 0x4a7, 0x4a7; 0x4a9, 0x4a9; 0x4ab, 0x4ab; 0x4ad, 0x4ad; 0x4af, 0x4af; 0x4b1, 0x4b1; 0x4b3, 0x4b3; 0x4b5, 0x4b5; 0x4b7, 0x4b7; 0x4b9, 0x4b9; 0x4bb, 0x4bb; 0x4bd, 0x4bd; 0x4bf, 0x4bf; 0x4c2, 0x4c2; 0x4c4, 0x4c4; 0x4c6, 0x4c6; 0x4c8, 0x4c8; 0x4ca, 0x4ca; 0x4cc, 0x4cc; 0x4ce, 0x4cf; 0x4d1, 0x4d1; 0x4d3, 0x4d3; 0x4d5, 0x4d5; 0x4d7, 0x4d7; 0x4d9, 0x4d9; 0x4db, 0x4db; 0x4dd, 0x4dd; 0x4df, 0x4df; 0x4e1, 0x4e1; 0x4e3, 0x4e3; 0x4e5, 0x4e5; 0x4e7, 0x4e7; 0x4e9, 0x4e9; 0x4eb, 0x4eb; 0x4ed, 0x4ed; 0x4ef, 0x4ef; 0x4f1, 0x4f1; 0x4f3, 0x4f3; 0x4f5, 0x4f5; 0x4f7, 0x4f7; 0x4f9, 0x4f9; 0x4fb, 0x4fb; 0x4fd, 0x4fd; 0x4ff, 0x4ff; 0x501, 0x501; 0x503, 0x503; 0x505, 0x505; 0x507, 0x507; 0x509, 0x509; 0x50b, 0x50b; 0x50d, 0x50d; 0x50f, 0x50f; 0x511, 0x511; 0x513, 0x513; 0x515, 0x515; 0x517, 0x517; 0x519, 0x519; 0x51b, 0x51b; 0x51d, 0x51d; 0x51f, 0x51f; 0x521, 0x521; 0x523, 0x523; 0x525, 0x525; 0x527, 0x527; 0x529, 0x529; 0x52b, 0x52b; 0x52d, 0x52d; 0x52f, 0x52f; 0x560, 0x588; 0x10d0, 0x10fa; 0x10fd, 0x10ff; 0x13f8, 0x13fd; 0x1c80, 0x1c88; 0x1c8a, 0x1c8a; 0x1d00, 0x1d2b; 0x1d6b, 0x1d77; 0x1d79, 0x1d9a; 0x1e01, 0x1e01; 0x1e03, 0x1e03; 0x1e05, 0x1e05; 0x1e07, 0x1e07; 0x1e09, 0x1e09; 0x1e0b, 0x1e0b; 0x1e0d, 0x1e0d; 0x1e0f, 0x1e0f; 0x1e11, 0x1e11; 0x1e13, 0x1e13; 0x1e15, 0x1e15; 0x1e17, 0x1e17; 0x1e19, 0x1e19; 0x1e1b, 0x1e1b; 0x1e1d, 0x1e1d; 0x1e1f, 0x1e1f; 0x1e21, 0x1e21; 0x1e23, 0x1e23; 0x1e25, 0x1e25; 0x1e27, 0x1e27; 0x1e29, 0x1e29; 0x1e2b, 0x1e2b; 0x1e2d, 0x1e2d; 0x1e2f, 0x1e2f; 0x1e31, 0x1e31; 0x1e33, 0x1e33; 0x1e35, 0x1e35; 0x1e37, 0x1e37; 0x1e39, 0x1e39; 0x1e3b, 0x1e3b; 0x1e3d, 0x1e3d; 0x1e3f, 0x1e3f; 0x1e41, 0x1e41; 0x1e43, 0x1e43; 0x1e45, 0x1e45; 0x1e47, 0x1e47; 0x1e49, 0x1e49; 0x1e4b, 0x1e4b; 0x1e4d, 0x1e4d; 0x1e4f, 0x1e4f; 0x1e51, 0x1e51; 0x1e53, 0x1e53; 0x1e55, 0x1e55; 0x1e57, 0x1e57; 0x1e59, 0x1e59; 0x1e5b, 0x1e5b; 0x1e5d, 0x1e5d; 0x1e5f, 0x1e5f; 0x1e61, 0x1e61; 0x1e63, 0x1e63; 0x1e65, 0x1e65; 0x1e67, 0x1e67; 0x1e69, 0x1e69; 0x1e6b, 0x1e6b; 0x1e6d, 0x1e6d; 0x1e6f, 0x1e6f; 0x1e71, 0x1e71; 0x1e73, 0x1e73; 0x1e75, 0x1e75; 0x1e77, 0x1e77; 0x1e79, 0x1e79; 0x1e7b, 0x1e7b; 0x1e7d, 0x1e7d; 0x1e7f, 0x1e7f; 0x1e81, 0x1e81; 0x1e83, 0x1e83; 0x1e85, 0x1e85; 0x1e87, 0x1e87; 0x1e89, 0x1e89; 0x1e8b, 0x1e8b; 0x1e8d, 0x1e8d; 0x1e8f, 0x1e8f; 0x1e91, 0x1e91; 0x1e93, 0x1e93; 0x1e95, 0x1e9d; 0x1e9f, 0x1e9f; 0x1ea1, 0x1ea1; 0x1ea3, 0x1ea3; 0x1ea5, 0x1ea5; 0x1ea7, 0x1ea7; 0x1ea9, 0x1ea9; 0x1eab, 0x1eab; 0x1ead, 0x1ead; 0x1eaf, 0x1eaf; 0x1eb1, 0x1eb1; 0x1eb3, 0x1eb3; 0x1eb5, 0x1eb5; 0x1eb7, 0x1eb7; 0x1eb9, 0x1eb9; 0x1ebb, 0x1ebb; 0x1ebd, 0x1ebd; 0x1ebf, 0x1ebf; 0x1ec1, 0x1ec1; 0x1ec3, 0x1ec3; 0x1ec5, 0x1ec5; 0x1ec7, 0x1ec7; 0x1ec9, 0x1ec9; 0x1ecb, 0x1ecb; 0x1ecd, 0x1ecd; 0x1ecf, 0x1ecf; 0x1ed1, 0x1ed1; 0x1ed3, 0x1ed3; 0x1ed5, 0x1ed5; 0x1ed7, 0x1ed7; 0x1ed9, 0x1ed9; 0x1edb, 0x1edb; 0x1edd, 0x1edd; 0x1edf, 0x1edf; 0x1ee1, 0x1ee1; 0x1ee3, 0x1ee3; 0x1ee5, 0x1ee5; 0x1ee7, 0x1ee7; 0x1ee9, 0x1ee9; 0x1eeb, 0x1eeb; 0x1eed, 0x1eed; 0x1eef, 0x1eef; 0x1ef1, 0x1ef1; 0x1ef3, 0x1ef3; 0x1ef5, 0x1ef5; 0x1ef7, 0x1ef7; 0x1ef9, 0x1ef9; 0x1efb, 0x1efb; 0x1efd, 0x1efd; 0x1eff, 0x1f07; 0x1f10, 0x1f15; 0x1f20, 0x1f27; 0x1f30, 0x1f37; 0x1f40, 0x1f45; 0x1f50, 0x1f57; 0x1f60, 0x1f67; 0x1f70, 0x1f7d; 0x1f80, 0x1f87; 0x1f90, 0x1f97; 0x1fa0, 0x1fa7; 0x1fb0, 0x1fb4; 0x1fb6, 0x1fb7; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fc7; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fd7; 0x1fe0, 0x1fe7; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ff7; 0x210a, 0x210a; 0x210e, 0x210f; 0x2113, 0x2113; 0x212f, 0x212f; 0x2134, 0x2134; 0x2139, 0x2139; 0x213c, 0x213d; 0x2146, 0x2149; 0x214e, 0x214e; 0x2184, 0x2184; 0x2c30, 0x2c5f; 0x2c61, 0x2c61; 0x2c65, 0x2c66; 0x2c68, 0x2c68; 0x2c6a, 0x2c6a; 0x2c6c, 0x2c6c; 0x2c71, 0x2c71; 0x2c73, 0x2c74; 0x2c76, 0x2c7b; 0x2c81, 0x2c81; 0x2c83, 0x2c83; 0x2c85, 0x2c85; 0x2c87, 0x2c87; 0x2c89, 0x2c89; 0x2c8b, 0x2c8b; 0x2c8d, 0x2c8d; 0x2c8f, 0x2c8f; 0x2c91, 0x2c91; 0x2c93, 0x2c93; 0x2c95, 0x2c95; 0x2c97, 0x2c97; 0x2c99, 0x2c99; 0x2c9b, 0x2c9b; 0x2c9d, 0x2c9d; 0x2c9f, 0x2c9f; 0x2ca1, 0x2ca1; 0x2ca3, 0x2ca3; 0x2ca5, 0x2ca5; 0x2ca7, 0x2ca7; 0x2ca9, 0x2ca9; 0x2cab, 0x2cab; 0x2cad, 0x2cad; 0x2caf, 0x2caf; 0x2cb1, 0x2cb1; 0x2cb3, 0x2cb3; 0x2cb5, 0x2cb5; 0x2cb7, 0x2cb7; 0x2cb9, 0x2cb9; 0x2cbb, 0x2cbb; 0x2cbd, 0x2cbd; 0x2cbf, 0x2cbf; 0x2cc1, 0x2cc1; 0x2cc3, 0x2cc3; 0x2cc5, 0x2cc5; 0x2cc7, 0x2cc7; 0x2cc9, 0x2cc9; 0x2ccb, 0x2ccb; 0x2ccd, 0x2ccd; 0x2ccf, 0x2ccf; 0x2cd1, 0x2cd1; 0x2cd3, 0x2cd3; 0x2cd5, 0x2cd5; 0x2cd7, 0x2cd7; 0x2cd9, 0x2cd9; 0x2cdb, 0x2cdb; 0x2cdd, 0x2cdd; 0x2cdf, 0x2cdf; 0x2ce1, 0x2ce1; 0x2ce3, 0x2ce4; 0x2cec, 0x2cec; 0x2cee, 0x2cee; 0x2cf3, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0xa641, 0xa641; 0xa643, 0xa643; 0xa645, 0xa645; 0xa647, 0xa647; 0xa649, 0xa649; 0xa64b, 0xa64b; 0xa64d, 0xa64d; 0xa64f, 0xa64f; 0xa651, 0xa651; 0xa653, 0xa653; 0xa655, 0xa655; 0xa657, 0xa657; 0xa659, 0xa659; 0xa65b, 0xa65b; 0xa65d, 0xa65d; 0xa65f, 0xa65f; 0xa661, 0xa661; 0xa663, 0xa663; 0xa665, 0xa665; 0xa667, 0xa667; 0xa669, 0xa669; 0xa66b, 0xa66b; 0xa66d, 0xa66d; 0xa681, 0xa681; 0xa683, 0xa683; 0xa685, 0xa685; 0xa687, 0xa687; 0xa689, 0xa689; 0xa68b, 0xa68b; 0xa68d, 0xa68d; 0xa68f, 0xa68f; 0xa691, 0xa691; 0xa693, 0xa693; 0xa695, 0xa695; 0xa697, 0xa697; 0xa699, 0xa699; 0xa69b, 0xa69b; 0xa723, 0xa723; 0xa725, 0xa725; 0xa727, 0xa727; 0xa729, 0xa729; 0xa72b, 0xa72b; 0xa72d, 0xa72d; 0xa72f, 0xa731; 0xa733, 0xa733; 0xa735, 0xa735; 0xa737, 0xa737; 0xa739, 0xa739; 0xa73b, 0xa73b; 0xa73d, 0xa73d; 0xa73f, 0xa73f; 0xa741, 0xa741; 0xa743, 0xa743; 0xa745, 0xa745; 0xa747, 0xa747; 0xa749, 0xa749; 0xa74b, 0xa74b; 0xa74d, 0xa74d; 0xa74f, 0xa74f; 0xa751, 0xa751; 0xa753, 0xa753; 0xa755, 0xa755; 0xa757, 0xa757; 0xa759, 0xa759; 0xa75b, 0xa75b; 0xa75d, 0xa75d; 0xa75f, 0xa75f; 0xa761, 0xa761; 0xa763, 0xa763; 0xa765, 0xa765; 0xa767, 0xa767; 0xa769, 0xa769; 0xa76b, 0xa76b; 0xa76d, 0xa76d; 0xa76f, 0xa76f; 0xa771, 0xa778; 0xa77a, 0xa77a; 0xa77c, 0xa77c; 0xa77f, 0xa77f; 0xa781, 0xa781; 0xa783, 0xa783; 0xa785, 0xa785; 0xa787, 0xa787; 0xa78c, 0xa78c; 0xa78e, 0xa78e; 0xa791, 0xa791; 0xa793, 0xa795; 0xa797, 0xa797; 0xa799, 0xa799; 0xa79b, 0xa79b; 0xa79d, 0xa79d; 0xa79f, 0xa79f; 0xa7a1, 0xa7a1; 0xa7a3, 0xa7a3; 0xa7a5, 0xa7a5; 0xa7a7, 0xa7a7; 0xa7a9, 0xa7a9; 0xa7af, 0xa7af; 0xa7b5, 0xa7b5; 0xa7b7, 0xa7b7; 0xa7b9, 0xa7b9; 0xa7bb, 0xa7bb; 0xa7bd, 0xa7bd; 0xa7bf, 0xa7bf; 0xa7c1, 0xa7c1; 0xa7c3, 0xa7c3; 0xa7c8, 0xa7c8; 0xa7ca, 0xa7ca; 0xa7cd, 0xa7cd; 0xa7cf, 0xa7cf; 0xa7d1, 0xa7d1; 0xa7d3, 0xa7d3; 0xa7d5, 0xa7d5; 0xa7d7, 0xa7d7; 0xa7d9, 0xa7d9; 0xa7db, 0xa7db; 0xa7f6, 0xa7f6; 0xa7fa, 0xa7fa; 0xab30, 0xab5a; 0xab60, 0xab68; 0xab70, 0xabbf; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xff41, 0xff5a; 0x10428, 0x1044f; 0x104d8, 0x104fb; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x10cc0, 0x10cf2; 0x10d70, 0x10d85; 0x118c0, 0x118df; 0x16e60, 0x16e7f; 0x16ebb, 0x16ed3; 0x1d41a, 0x1d433; 0x1d44e, 0x1d454; 0x1d456, 0x1d467; 0x1d482, 0x1d49b; 0x1d4b6, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d4cf; 0x1d4ea, 0x1d503; 0x1d51e, 0x1d537; 0x1d552, 0x1d56b; 0x1d586, 0x1d59f; 0x1d5ba, 0x1d5d3; 0x1d5ee, 0x1d607; 0x1d622, 0x1d63b; 0x1d656, 0x1d66f; 0x1d68a, 0x1d6a5; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6e1; 0x1d6fc, 0x1d714; 0x1d716, 0x1d71b; 0x1d736, 0x1d74e; 0x1d750, 0x1d755; 0x1d770, 0x1d788; 0x1d78a, 0x1d78f; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7c9; 0x1d7cb, 0x1d7cb; 0x1df00, 0x1df09; 0x1df0b, 0x1df1e; 0x1df25, 0x1df2a; 0x1e922, 0x1e943] let lm = Cset.of_list [0x2b0, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x374, 0x374; 0x37a, 0x37a; 0x559, 0x559; 0x640, 0x640; 0x6e5, 0x6e6; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x81a, 0x81a; 0x824, 0x824; 0x828, 0x828; 0x8c9, 0x8c9; 0x971, 0x971; 0xe46, 0xe46; 0xec6, 0xec6; 0x10fc, 0x10fc; 0x17d7, 0x17d7; 0x1843, 0x1843; 0x1aa7, 0x1aa7; 0x1c78, 0x1c7d; 0x1d2c, 0x1d6a; 0x1d78, 0x1d78; 0x1d9b, 0x1dbf; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x2c7c, 0x2c7d; 0x2d6f, 0x2d6f; 0x2e2f, 0x2e2f; 0x3005, 0x3005; 0x3031, 0x3035; 0x303b, 0x303b; 0x309d, 0x309e; 0x30fc, 0x30fe; 0xa015, 0xa015; 0xa4f8, 0xa4fd; 0xa60c, 0xa60c; 0xa67f, 0xa67f; 0xa69c, 0xa69d; 0xa717, 0xa71f; 0xa770, 0xa770; 0xa788, 0xa788; 0xa7f1, 0xa7f4; 0xa7f8, 0xa7f9; 0xa9cf, 0xa9cf; 0xa9e6, 0xa9e6; 0xaa70, 0xaa70; 0xaadd, 0xaadd; 0xaaf3, 0xaaf4; 0xab5c, 0xab5f; 0xab69, 0xab69; 0xff70, 0xff70; 0xff9e, 0xff9f; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10d4e, 0x10d4e; 0x10d6f, 0x10d6f; 0x10ec5, 0x10ec5; 0x11dd9, 0x11dd9; 0x16b40, 0x16b43; 0x16d40, 0x16d42; 0x16d6b, 0x16d6c; 0x16f93, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff2, 0x16ff3; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1e030, 0x1e06d; 0x1e137, 0x1e13d; 0x1e4eb, 0x1e4eb; 0x1e6ff, 0x1e6ff; 0x1e94b, 0x1e94b] let lo = Cset.of_list [0xaa, 0xaa; 0xba, 0xba; 0x1bb, 0x1bb; 0x1c0, 0x1c3; 0x294, 0x295; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x620, 0x63f; 0x641, 0x64a; 0x66e, 0x66f; 0x671, 0x6d3; 0x6d5, 0x6d5; 0x6ee, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x710; 0x712, 0x72f; 0x74d, 0x7a5; 0x7b1, 0x7b1; 0x7ca, 0x7ea; 0x800, 0x815; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x8a0, 0x8c8; 0x904, 0x939; 0x93d, 0x93d; 0x950, 0x950; 0x958, 0x961; 0x972, 0x980; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9bd; 0x9ce, 0x9ce; 0x9dc, 0x9dd; 0x9df, 0x9e1; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa72, 0xa74; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xabd; 0xad0, 0xad0; 0xae0, 0xae1; 0xaf9, 0xaf9; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb3d; 0xb5c, 0xb5d; 0xb5f, 0xb61; 0xb71, 0xb71; 0xb83, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbd0, 0xbd0; 0xc05, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc3d; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc61; 0xc80, 0xc80; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcbd; 0xcdc, 0xcde; 0xce0, 0xce1; 0xcf1, 0xcf2; 0xd04, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd3d; 0xd4e, 0xd4e; 0xd54, 0xd56; 0xd5f, 0xd61; 0xd7a, 0xd7f; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xe01, 0xe30; 0xe32, 0xe33; 0xe40, 0xe45; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb0; 0xeb2, 0xeb3; 0xebd, 0xebd; 0xec0, 0xec4; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf88, 0xf8c; 0x1000, 0x102a; 0x103f, 0x103f; 0x1050, 0x1055; 0x105a, 0x105d; 0x1061, 0x1061; 0x1065, 0x1066; 0x106e, 0x1070; 0x1075, 0x1081; 0x108e, 0x108e; 0x1100, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16f1, 0x16f8; 0x1700, 0x1711; 0x171f, 0x1731; 0x1740, 0x1751; 0x1760, 0x176c; 0x176e, 0x1770; 0x1780, 0x17b3; 0x17dc, 0x17dc; 0x1820, 0x1842; 0x1844, 0x1878; 0x1880, 0x1884; 0x1887, 0x18a8; 0x18aa, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a16; 0x1a20, 0x1a54; 0x1b05, 0x1b33; 0x1b45, 0x1b4c; 0x1b83, 0x1ba0; 0x1bae, 0x1baf; 0x1bba, 0x1be5; 0x1c00, 0x1c23; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c77; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x2135, 0x2138; 0x2d30, 0x2d67; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x3006, 0x3006; 0x303c, 0x303c; 0x3041, 0x3096; 0x309f, 0x309f; 0x30a1, 0x30fa; 0x30ff, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa014; 0xa016, 0xa48c; 0xa4d0, 0xa4f7; 0xa500, 0xa60b; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa66e, 0xa66e; 0xa6a0, 0xa6e5; 0xa78f, 0xa78f; 0xa7f7, 0xa7f7; 0xa7fb, 0xa801; 0xa803, 0xa805; 0xa807, 0xa80a; 0xa80c, 0xa822; 0xa840, 0xa873; 0xa882, 0xa8b3; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8fe; 0xa90a, 0xa925; 0xa930, 0xa946; 0xa960, 0xa97c; 0xa984, 0xa9b2; 0xa9e0, 0xa9e4; 0xa9e7, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa28; 0xaa40, 0xaa42; 0xaa44, 0xaa4b; 0xaa60, 0xaa6f; 0xaa71, 0xaa76; 0xaa7a, 0xaa7a; 0xaa7e, 0xaaaf; 0xaab1, 0xaab1; 0xaab5, 0xaab6; 0xaab9, 0xaabd; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadc; 0xaae0, 0xaaea; 0xaaf2, 0xaaf2; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xabc0, 0xabe2; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb1d, 0xfb1d; 0xfb1f, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff66, 0xff6f; 0xff71, 0xff9d; 0xffa0, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x10340; 0x10342, 0x10349; 0x10350, 0x10375; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x10450, 0x1049d; 0x10500, 0x10527; 0x10530, 0x10563; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a00; 0x10a10, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10d00, 0x10d23; 0x10d4a, 0x10d4d; 0x10d4f, 0x10d4f; 0x10e80, 0x10ea9; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec4; 0x10ec6, 0x10ec7; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11003, 0x11037; 0x11071, 0x11072; 0x11075, 0x11075; 0x11083, 0x110af; 0x110d0, 0x110e8; 0x11103, 0x11126; 0x11144, 0x11144; 0x11147, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11183, 0x111b2; 0x111c1, 0x111c4; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x1122b; 0x1123f, 0x11240; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112de; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x1133d; 0x11350, 0x11350; 0x1135d, 0x11361; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113b7; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11434; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114af; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115ae; 0x115d8, 0x115db; 0x11600, 0x1162f; 0x11644, 0x11644; 0x11680, 0x116aa; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x11740, 0x11746; 0x11800, 0x1182b; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x1192f; 0x1193f, 0x1193f; 0x11941, 0x11941; 0x119a0, 0x119a7; 0x119aa, 0x119d0; 0x119e1, 0x119e1; 0x119e3, 0x119e3; 0x11a00, 0x11a00; 0x11a0b, 0x11a32; 0x11a3a, 0x11a3a; 0x11a50, 0x11a50; 0x11a5c, 0x11a89; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c2e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d30; 0x11d46, 0x11d46; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d89; 0x11d98, 0x11d98; 0x11db0, 0x11dd8; 0x11dda, 0x11ddb; 0x11ee0, 0x11ef2; 0x11f02, 0x11f02; 0x11f04, 0x11f10; 0x11f12, 0x11f33; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1611d; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d43, 0x16d6a; 0x16f00, 0x16f4a; 0x16f50, 0x16f50; 0x17000, 0x18cd5; 0x18cff, 0x18d1e; 0x18d80, 0x18df2; 0x1b000, 0x1b122; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b167; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1df0a, 0x1df0a; 0x1e100, 0x1e12c; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4ea; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6e2; 0x1e6e4, 0x1e6e5; 0x1e6e7, 0x1e6ed; 0x1e6f0, 0x1e6f4; 0x1e6fe, 0x1e6fe; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x20000, 0x2a6df; 0x2a700, 0x2b81d; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479] let lt = Cset.of_list [0x1c5, 0x1c5; 0x1c8, 0x1c8; 0x1cb, 0x1cb; 0x1f2, 0x1f2; 0x1f88, 0x1f8f; 0x1f98, 0x1f9f; 0x1fa8, 0x1faf; 0x1fbc, 0x1fbc; 0x1fcc, 0x1fcc; 0x1ffc, 0x1ffc] let lu = Cset.of_list [0x41, 0x5a; 0xc0, 0xd6; 0xd8, 0xde; 0x100, 0x100; 0x102, 0x102; 0x104, 0x104; 0x106, 0x106; 0x108, 0x108; 0x10a, 0x10a; 0x10c, 0x10c; 0x10e, 0x10e; 0x110, 0x110; 0x112, 0x112; 0x114, 0x114; 0x116, 0x116; 0x118, 0x118; 0x11a, 0x11a; 0x11c, 0x11c; 0x11e, 0x11e; 0x120, 0x120; 0x122, 0x122; 0x124, 0x124; 0x126, 0x126; 0x128, 0x128; 0x12a, 0x12a; 0x12c, 0x12c; 0x12e, 0x12e; 0x130, 0x130; 0x132, 0x132; 0x134, 0x134; 0x136, 0x136; 0x139, 0x139; 0x13b, 0x13b; 0x13d, 0x13d; 0x13f, 0x13f; 0x141, 0x141; 0x143, 0x143; 0x145, 0x145; 0x147, 0x147; 0x14a, 0x14a; 0x14c, 0x14c; 0x14e, 0x14e; 0x150, 0x150; 0x152, 0x152; 0x154, 0x154; 0x156, 0x156; 0x158, 0x158; 0x15a, 0x15a; 0x15c, 0x15c; 0x15e, 0x15e; 0x160, 0x160; 0x162, 0x162; 0x164, 0x164; 0x166, 0x166; 0x168, 0x168; 0x16a, 0x16a; 0x16c, 0x16c; 0x16e, 0x16e; 0x170, 0x170; 0x172, 0x172; 0x174, 0x174; 0x176, 0x176; 0x178, 0x179; 0x17b, 0x17b; 0x17d, 0x17d; 0x181, 0x182; 0x184, 0x184; 0x186, 0x187; 0x189, 0x18b; 0x18e, 0x191; 0x193, 0x194; 0x196, 0x198; 0x19c, 0x19d; 0x19f, 0x1a0; 0x1a2, 0x1a2; 0x1a4, 0x1a4; 0x1a6, 0x1a7; 0x1a9, 0x1a9; 0x1ac, 0x1ac; 0x1ae, 0x1af; 0x1b1, 0x1b3; 0x1b5, 0x1b5; 0x1b7, 0x1b8; 0x1bc, 0x1bc; 0x1c4, 0x1c4; 0x1c7, 0x1c7; 0x1ca, 0x1ca; 0x1cd, 0x1cd; 0x1cf, 0x1cf; 0x1d1, 0x1d1; 0x1d3, 0x1d3; 0x1d5, 0x1d5; 0x1d7, 0x1d7; 0x1d9, 0x1d9; 0x1db, 0x1db; 0x1de, 0x1de; 0x1e0, 0x1e0; 0x1e2, 0x1e2; 0x1e4, 0x1e4; 0x1e6, 0x1e6; 0x1e8, 0x1e8; 0x1ea, 0x1ea; 0x1ec, 0x1ec; 0x1ee, 0x1ee; 0x1f1, 0x1f1; 0x1f4, 0x1f4; 0x1f6, 0x1f8; 0x1fa, 0x1fa; 0x1fc, 0x1fc; 0x1fe, 0x1fe; 0x200, 0x200; 0x202, 0x202; 0x204, 0x204; 0x206, 0x206; 0x208, 0x208; 0x20a, 0x20a; 0x20c, 0x20c; 0x20e, 0x20e; 0x210, 0x210; 0x212, 0x212; 0x214, 0x214; 0x216, 0x216; 0x218, 0x218; 0x21a, 0x21a; 0x21c, 0x21c; 0x21e, 0x21e; 0x220, 0x220; 0x222, 0x222; 0x224, 0x224; 0x226, 0x226; 0x228, 0x228; 0x22a, 0x22a; 0x22c, 0x22c; 0x22e, 0x22e; 0x230, 0x230; 0x232, 0x232; 0x23a, 0x23b; 0x23d, 0x23e; 0x241, 0x241; 0x243, 0x246; 0x248, 0x248; 0x24a, 0x24a; 0x24c, 0x24c; 0x24e, 0x24e; 0x370, 0x370; 0x372, 0x372; 0x376, 0x376; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x38f; 0x391, 0x3a1; 0x3a3, 0x3ab; 0x3cf, 0x3cf; 0x3d2, 0x3d4; 0x3d8, 0x3d8; 0x3da, 0x3da; 0x3dc, 0x3dc; 0x3de, 0x3de; 0x3e0, 0x3e0; 0x3e2, 0x3e2; 0x3e4, 0x3e4; 0x3e6, 0x3e6; 0x3e8, 0x3e8; 0x3ea, 0x3ea; 0x3ec, 0x3ec; 0x3ee, 0x3ee; 0x3f4, 0x3f4; 0x3f7, 0x3f7; 0x3f9, 0x3fa; 0x3fd, 0x42f; 0x460, 0x460; 0x462, 0x462; 0x464, 0x464; 0x466, 0x466; 0x468, 0x468; 0x46a, 0x46a; 0x46c, 0x46c; 0x46e, 0x46e; 0x470, 0x470; 0x472, 0x472; 0x474, 0x474; 0x476, 0x476; 0x478, 0x478; 0x47a, 0x47a; 0x47c, 0x47c; 0x47e, 0x47e; 0x480, 0x480; 0x48a, 0x48a; 0x48c, 0x48c; 0x48e, 0x48e; 0x490, 0x490; 0x492, 0x492; 0x494, 0x494; 0x496, 0x496; 0x498, 0x498; 0x49a, 0x49a; 0x49c, 0x49c; 0x49e, 0x49e; 0x4a0, 0x4a0; 0x4a2, 0x4a2; 0x4a4, 0x4a4; 0x4a6, 0x4a6; 0x4a8, 0x4a8; 0x4aa, 0x4aa; 0x4ac, 0x4ac; 0x4ae, 0x4ae; 0x4b0, 0x4b0; 0x4b2, 0x4b2; 0x4b4, 0x4b4; 0x4b6, 0x4b6; 0x4b8, 0x4b8; 0x4ba, 0x4ba; 0x4bc, 0x4bc; 0x4be, 0x4be; 0x4c0, 0x4c1; 0x4c3, 0x4c3; 0x4c5, 0x4c5; 0x4c7, 0x4c7; 0x4c9, 0x4c9; 0x4cb, 0x4cb; 0x4cd, 0x4cd; 0x4d0, 0x4d0; 0x4d2, 0x4d2; 0x4d4, 0x4d4; 0x4d6, 0x4d6; 0x4d8, 0x4d8; 0x4da, 0x4da; 0x4dc, 0x4dc; 0x4de, 0x4de; 0x4e0, 0x4e0; 0x4e2, 0x4e2; 0x4e4, 0x4e4; 0x4e6, 0x4e6; 0x4e8, 0x4e8; 0x4ea, 0x4ea; 0x4ec, 0x4ec; 0x4ee, 0x4ee; 0x4f0, 0x4f0; 0x4f2, 0x4f2; 0x4f4, 0x4f4; 0x4f6, 0x4f6; 0x4f8, 0x4f8; 0x4fa, 0x4fa; 0x4fc, 0x4fc; 0x4fe, 0x4fe; 0x500, 0x500; 0x502, 0x502; 0x504, 0x504; 0x506, 0x506; 0x508, 0x508; 0x50a, 0x50a; 0x50c, 0x50c; 0x50e, 0x50e; 0x510, 0x510; 0x512, 0x512; 0x514, 0x514; 0x516, 0x516; 0x518, 0x518; 0x51a, 0x51a; 0x51c, 0x51c; 0x51e, 0x51e; 0x520, 0x520; 0x522, 0x522; 0x524, 0x524; 0x526, 0x526; 0x528, 0x528; 0x52a, 0x52a; 0x52c, 0x52c; 0x52e, 0x52e; 0x531, 0x556; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x13a0, 0x13f5; 0x1c89, 0x1c89; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1e00, 0x1e00; 0x1e02, 0x1e02; 0x1e04, 0x1e04; 0x1e06, 0x1e06; 0x1e08, 0x1e08; 0x1e0a, 0x1e0a; 0x1e0c, 0x1e0c; 0x1e0e, 0x1e0e; 0x1e10, 0x1e10; 0x1e12, 0x1e12; 0x1e14, 0x1e14; 0x1e16, 0x1e16; 0x1e18, 0x1e18; 0x1e1a, 0x1e1a; 0x1e1c, 0x1e1c; 0x1e1e, 0x1e1e; 0x1e20, 0x1e20; 0x1e22, 0x1e22; 0x1e24, 0x1e24; 0x1e26, 0x1e26; 0x1e28, 0x1e28; 0x1e2a, 0x1e2a; 0x1e2c, 0x1e2c; 0x1e2e, 0x1e2e; 0x1e30, 0x1e30; 0x1e32, 0x1e32; 0x1e34, 0x1e34; 0x1e36, 0x1e36; 0x1e38, 0x1e38; 0x1e3a, 0x1e3a; 0x1e3c, 0x1e3c; 0x1e3e, 0x1e3e; 0x1e40, 0x1e40; 0x1e42, 0x1e42; 0x1e44, 0x1e44; 0x1e46, 0x1e46; 0x1e48, 0x1e48; 0x1e4a, 0x1e4a; 0x1e4c, 0x1e4c; 0x1e4e, 0x1e4e; 0x1e50, 0x1e50; 0x1e52, 0x1e52; 0x1e54, 0x1e54; 0x1e56, 0x1e56; 0x1e58, 0x1e58; 0x1e5a, 0x1e5a; 0x1e5c, 0x1e5c; 0x1e5e, 0x1e5e; 0x1e60, 0x1e60; 0x1e62, 0x1e62; 0x1e64, 0x1e64; 0x1e66, 0x1e66; 0x1e68, 0x1e68; 0x1e6a, 0x1e6a; 0x1e6c, 0x1e6c; 0x1e6e, 0x1e6e; 0x1e70, 0x1e70; 0x1e72, 0x1e72; 0x1e74, 0x1e74; 0x1e76, 0x1e76; 0x1e78, 0x1e78; 0x1e7a, 0x1e7a; 0x1e7c, 0x1e7c; 0x1e7e, 0x1e7e; 0x1e80, 0x1e80; 0x1e82, 0x1e82; 0x1e84, 0x1e84; 0x1e86, 0x1e86; 0x1e88, 0x1e88; 0x1e8a, 0x1e8a; 0x1e8c, 0x1e8c; 0x1e8e, 0x1e8e; 0x1e90, 0x1e90; 0x1e92, 0x1e92; 0x1e94, 0x1e94; 0x1e9e, 0x1e9e; 0x1ea0, 0x1ea0; 0x1ea2, 0x1ea2; 0x1ea4, 0x1ea4; 0x1ea6, 0x1ea6; 0x1ea8, 0x1ea8; 0x1eaa, 0x1eaa; 0x1eac, 0x1eac; 0x1eae, 0x1eae; 0x1eb0, 0x1eb0; 0x1eb2, 0x1eb2; 0x1eb4, 0x1eb4; 0x1eb6, 0x1eb6; 0x1eb8, 0x1eb8; 0x1eba, 0x1eba; 0x1ebc, 0x1ebc; 0x1ebe, 0x1ebe; 0x1ec0, 0x1ec0; 0x1ec2, 0x1ec2; 0x1ec4, 0x1ec4; 0x1ec6, 0x1ec6; 0x1ec8, 0x1ec8; 0x1eca, 0x1eca; 0x1ecc, 0x1ecc; 0x1ece, 0x1ece; 0x1ed0, 0x1ed0; 0x1ed2, 0x1ed2; 0x1ed4, 0x1ed4; 0x1ed6, 0x1ed6; 0x1ed8, 0x1ed8; 0x1eda, 0x1eda; 0x1edc, 0x1edc; 0x1ede, 0x1ede; 0x1ee0, 0x1ee0; 0x1ee2, 0x1ee2; 0x1ee4, 0x1ee4; 0x1ee6, 0x1ee6; 0x1ee8, 0x1ee8; 0x1eea, 0x1eea; 0x1eec, 0x1eec; 0x1eee, 0x1eee; 0x1ef0, 0x1ef0; 0x1ef2, 0x1ef2; 0x1ef4, 0x1ef4; 0x1ef6, 0x1ef6; 0x1ef8, 0x1ef8; 0x1efa, 0x1efa; 0x1efc, 0x1efc; 0x1efe, 0x1efe; 0x1f08, 0x1f0f; 0x1f18, 0x1f1d; 0x1f28, 0x1f2f; 0x1f38, 0x1f3f; 0x1f48, 0x1f4d; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f5f; 0x1f68, 0x1f6f; 0x1fb8, 0x1fbb; 0x1fc8, 0x1fcb; 0x1fd8, 0x1fdb; 0x1fe8, 0x1fec; 0x1ff8, 0x1ffb; 0x2102, 0x2102; 0x2107, 0x2107; 0x210b, 0x210d; 0x2110, 0x2112; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x212d; 0x2130, 0x2133; 0x213e, 0x213f; 0x2145, 0x2145; 0x2183, 0x2183; 0x2c00, 0x2c2f; 0x2c60, 0x2c60; 0x2c62, 0x2c64; 0x2c67, 0x2c67; 0x2c69, 0x2c69; 0x2c6b, 0x2c6b; 0x2c6d, 0x2c70; 0x2c72, 0x2c72; 0x2c75, 0x2c75; 0x2c7e, 0x2c80; 0x2c82, 0x2c82; 0x2c84, 0x2c84; 0x2c86, 0x2c86; 0x2c88, 0x2c88; 0x2c8a, 0x2c8a; 0x2c8c, 0x2c8c; 0x2c8e, 0x2c8e; 0x2c90, 0x2c90; 0x2c92, 0x2c92; 0x2c94, 0x2c94; 0x2c96, 0x2c96; 0x2c98, 0x2c98; 0x2c9a, 0x2c9a; 0x2c9c, 0x2c9c; 0x2c9e, 0x2c9e; 0x2ca0, 0x2ca0; 0x2ca2, 0x2ca2; 0x2ca4, 0x2ca4; 0x2ca6, 0x2ca6; 0x2ca8, 0x2ca8; 0x2caa, 0x2caa; 0x2cac, 0x2cac; 0x2cae, 0x2cae; 0x2cb0, 0x2cb0; 0x2cb2, 0x2cb2; 0x2cb4, 0x2cb4; 0x2cb6, 0x2cb6; 0x2cb8, 0x2cb8; 0x2cba, 0x2cba; 0x2cbc, 0x2cbc; 0x2cbe, 0x2cbe; 0x2cc0, 0x2cc0; 0x2cc2, 0x2cc2; 0x2cc4, 0x2cc4; 0x2cc6, 0x2cc6; 0x2cc8, 0x2cc8; 0x2cca, 0x2cca; 0x2ccc, 0x2ccc; 0x2cce, 0x2cce; 0x2cd0, 0x2cd0; 0x2cd2, 0x2cd2; 0x2cd4, 0x2cd4; 0x2cd6, 0x2cd6; 0x2cd8, 0x2cd8; 0x2cda, 0x2cda; 0x2cdc, 0x2cdc; 0x2cde, 0x2cde; 0x2ce0, 0x2ce0; 0x2ce2, 0x2ce2; 0x2ceb, 0x2ceb; 0x2ced, 0x2ced; 0x2cf2, 0x2cf2; 0xa640, 0xa640; 0xa642, 0xa642; 0xa644, 0xa644; 0xa646, 0xa646; 0xa648, 0xa648; 0xa64a, 0xa64a; 0xa64c, 0xa64c; 0xa64e, 0xa64e; 0xa650, 0xa650; 0xa652, 0xa652; 0xa654, 0xa654; 0xa656, 0xa656; 0xa658, 0xa658; 0xa65a, 0xa65a; 0xa65c, 0xa65c; 0xa65e, 0xa65e; 0xa660, 0xa660; 0xa662, 0xa662; 0xa664, 0xa664; 0xa666, 0xa666; 0xa668, 0xa668; 0xa66a, 0xa66a; 0xa66c, 0xa66c; 0xa680, 0xa680; 0xa682, 0xa682; 0xa684, 0xa684; 0xa686, 0xa686; 0xa688, 0xa688; 0xa68a, 0xa68a; 0xa68c, 0xa68c; 0xa68e, 0xa68e; 0xa690, 0xa690; 0xa692, 0xa692; 0xa694, 0xa694; 0xa696, 0xa696; 0xa698, 0xa698; 0xa69a, 0xa69a; 0xa722, 0xa722; 0xa724, 0xa724; 0xa726, 0xa726; 0xa728, 0xa728; 0xa72a, 0xa72a; 0xa72c, 0xa72c; 0xa72e, 0xa72e; 0xa732, 0xa732; 0xa734, 0xa734; 0xa736, 0xa736; 0xa738, 0xa738; 0xa73a, 0xa73a; 0xa73c, 0xa73c; 0xa73e, 0xa73e; 0xa740, 0xa740; 0xa742, 0xa742; 0xa744, 0xa744; 0xa746, 0xa746; 0xa748, 0xa748; 0xa74a, 0xa74a; 0xa74c, 0xa74c; 0xa74e, 0xa74e; 0xa750, 0xa750; 0xa752, 0xa752; 0xa754, 0xa754; 0xa756, 0xa756; 0xa758, 0xa758; 0xa75a, 0xa75a; 0xa75c, 0xa75c; 0xa75e, 0xa75e; 0xa760, 0xa760; 0xa762, 0xa762; 0xa764, 0xa764; 0xa766, 0xa766; 0xa768, 0xa768; 0xa76a, 0xa76a; 0xa76c, 0xa76c; 0xa76e, 0xa76e; 0xa779, 0xa779; 0xa77b, 0xa77b; 0xa77d, 0xa77e; 0xa780, 0xa780; 0xa782, 0xa782; 0xa784, 0xa784; 0xa786, 0xa786; 0xa78b, 0xa78b; 0xa78d, 0xa78d; 0xa790, 0xa790; 0xa792, 0xa792; 0xa796, 0xa796; 0xa798, 0xa798; 0xa79a, 0xa79a; 0xa79c, 0xa79c; 0xa79e, 0xa79e; 0xa7a0, 0xa7a0; 0xa7a2, 0xa7a2; 0xa7a4, 0xa7a4; 0xa7a6, 0xa7a6; 0xa7a8, 0xa7a8; 0xa7aa, 0xa7ae; 0xa7b0, 0xa7b4; 0xa7b6, 0xa7b6; 0xa7b8, 0xa7b8; 0xa7ba, 0xa7ba; 0xa7bc, 0xa7bc; 0xa7be, 0xa7be; 0xa7c0, 0xa7c0; 0xa7c2, 0xa7c2; 0xa7c4, 0xa7c7; 0xa7c9, 0xa7c9; 0xa7cb, 0xa7cc; 0xa7ce, 0xa7ce; 0xa7d0, 0xa7d0; 0xa7d2, 0xa7d2; 0xa7d4, 0xa7d4; 0xa7d6, 0xa7d6; 0xa7d8, 0xa7d8; 0xa7da, 0xa7da; 0xa7dc, 0xa7dc; 0xa7f5, 0xa7f5; 0xff21, 0xff3a; 0x10400, 0x10427; 0x104b0, 0x104d3; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10c80, 0x10cb2; 0x10d50, 0x10d65; 0x118a0, 0x118bf; 0x16e40, 0x16e5f; 0x16ea0, 0x16eb8; 0x1d400, 0x1d419; 0x1d434, 0x1d44d; 0x1d468, 0x1d481; 0x1d49c, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b5; 0x1d4d0, 0x1d4e9; 0x1d504, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d538, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d56c, 0x1d585; 0x1d5a0, 0x1d5b9; 0x1d5d4, 0x1d5ed; 0x1d608, 0x1d621; 0x1d63c, 0x1d655; 0x1d670, 0x1d689; 0x1d6a8, 0x1d6c0; 0x1d6e2, 0x1d6fa; 0x1d71c, 0x1d734; 0x1d756, 0x1d76e; 0x1d790, 0x1d7a8; 0x1d7ca, 0x1d7ca; 0x1e900, 0x1e921] let mc = Cset.of_list [0x903, 0x903; 0x93b, 0x93b; 0x93e, 0x940; 0x949, 0x94c; 0x94e, 0x94f; 0x982, 0x983; 0x9be, 0x9c0; 0x9c7, 0x9c8; 0x9cb, 0x9cc; 0x9d7, 0x9d7; 0xa03, 0xa03; 0xa3e, 0xa40; 0xa83, 0xa83; 0xabe, 0xac0; 0xac9, 0xac9; 0xacb, 0xacc; 0xb02, 0xb03; 0xb3e, 0xb3e; 0xb40, 0xb40; 0xb47, 0xb48; 0xb4b, 0xb4c; 0xb57, 0xb57; 0xbbe, 0xbbf; 0xbc1, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcc; 0xbd7, 0xbd7; 0xc01, 0xc03; 0xc41, 0xc44; 0xc82, 0xc83; 0xcbe, 0xcbe; 0xcc0, 0xcc4; 0xcc7, 0xcc8; 0xcca, 0xccb; 0xcd5, 0xcd6; 0xcf3, 0xcf3; 0xd02, 0xd03; 0xd3e, 0xd40; 0xd46, 0xd48; 0xd4a, 0xd4c; 0xd57, 0xd57; 0xd82, 0xd83; 0xdcf, 0xdd1; 0xdd8, 0xddf; 0xdf2, 0xdf3; 0xf3e, 0xf3f; 0xf7f, 0xf7f; 0x102b, 0x102c; 0x1031, 0x1031; 0x1038, 0x1038; 0x103b, 0x103c; 0x1056, 0x1057; 0x1062, 0x1064; 0x1067, 0x106d; 0x1083, 0x1084; 0x1087, 0x108c; 0x108f, 0x108f; 0x109a, 0x109c; 0x1715, 0x1715; 0x1734, 0x1734; 0x17b6, 0x17b6; 0x17be, 0x17c5; 0x17c7, 0x17c8; 0x1923, 0x1926; 0x1929, 0x192b; 0x1930, 0x1931; 0x1933, 0x1938; 0x1a19, 0x1a1a; 0x1a55, 0x1a55; 0x1a57, 0x1a57; 0x1a61, 0x1a61; 0x1a63, 0x1a64; 0x1a6d, 0x1a72; 0x1b04, 0x1b04; 0x1b35, 0x1b35; 0x1b3b, 0x1b3b; 0x1b3d, 0x1b41; 0x1b43, 0x1b44; 0x1b82, 0x1b82; 0x1ba1, 0x1ba1; 0x1ba6, 0x1ba7; 0x1baa, 0x1baa; 0x1be7, 0x1be7; 0x1bea, 0x1bec; 0x1bee, 0x1bee; 0x1bf2, 0x1bf3; 0x1c24, 0x1c2b; 0x1c34, 0x1c35; 0x1ce1, 0x1ce1; 0x1cf7, 0x1cf7; 0x302e, 0x302f; 0xa823, 0xa824; 0xa827, 0xa827; 0xa880, 0xa881; 0xa8b4, 0xa8c3; 0xa952, 0xa953; 0xa983, 0xa983; 0xa9b4, 0xa9b5; 0xa9ba, 0xa9bb; 0xa9be, 0xa9c0; 0xaa2f, 0xaa30; 0xaa33, 0xaa34; 0xaa4d, 0xaa4d; 0xaa7b, 0xaa7b; 0xaa7d, 0xaa7d; 0xaaeb, 0xaaeb; 0xaaee, 0xaaef; 0xaaf5, 0xaaf5; 0xabe3, 0xabe4; 0xabe6, 0xabe7; 0xabe9, 0xabea; 0xabec, 0xabec; 0x11000, 0x11000; 0x11002, 0x11002; 0x11082, 0x11082; 0x110b0, 0x110b2; 0x110b7, 0x110b8; 0x1112c, 0x1112c; 0x11145, 0x11146; 0x11182, 0x11182; 0x111b3, 0x111b5; 0x111bf, 0x111c0; 0x111ce, 0x111ce; 0x1122c, 0x1122e; 0x11232, 0x11233; 0x11235, 0x11235; 0x112e0, 0x112e2; 0x11302, 0x11303; 0x1133e, 0x1133f; 0x11341, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134d; 0x11357, 0x11357; 0x11362, 0x11363; 0x113b8, 0x113ba; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113cd; 0x113cf, 0x113cf; 0x11435, 0x11437; 0x11440, 0x11441; 0x11445, 0x11445; 0x114b0, 0x114b2; 0x114b9, 0x114b9; 0x114bb, 0x114be; 0x114c1, 0x114c1; 0x115af, 0x115b1; 0x115b8, 0x115bb; 0x115be, 0x115be; 0x11630, 0x11632; 0x1163b, 0x1163c; 0x1163e, 0x1163e; 0x116ac, 0x116ac; 0x116ae, 0x116af; 0x116b6, 0x116b6; 0x1171e, 0x1171e; 0x11720, 0x11721; 0x11726, 0x11726; 0x1182c, 0x1182e; 0x11838, 0x11838; 0x11930, 0x11935; 0x11937, 0x11938; 0x1193d, 0x1193d; 0x11940, 0x11940; 0x11942, 0x11942; 0x119d1, 0x119d3; 0x119dc, 0x119df; 0x119e4, 0x119e4; 0x11a39, 0x11a39; 0x11a57, 0x11a58; 0x11a97, 0x11a97; 0x11b61, 0x11b61; 0x11b65, 0x11b65; 0x11b67, 0x11b67; 0x11c2f, 0x11c2f; 0x11c3e, 0x11c3e; 0x11ca9, 0x11ca9; 0x11cb1, 0x11cb1; 0x11cb4, 0x11cb4; 0x11d8a, 0x11d8e; 0x11d93, 0x11d94; 0x11d96, 0x11d96; 0x11ef5, 0x11ef6; 0x11f03, 0x11f03; 0x11f34, 0x11f35; 0x11f3e, 0x11f3f; 0x11f41, 0x11f41; 0x1612a, 0x1612c; 0x16f51, 0x16f87; 0x16ff0, 0x16ff1; 0x1d165, 0x1d166; 0x1d16d, 0x1d172] let me = Cset.of_list [0x488, 0x489; 0x1abe, 0x1abe; 0x20dd, 0x20e0; 0x20e2, 0x20e4; 0xa670, 0xa672] let mn = Cset.of_list [0x300, 0x36f; 0x483, 0x487; 0x591, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c7; 0x610, 0x61a; 0x64b, 0x65f; 0x670, 0x670; 0x6d6, 0x6dc; 0x6df, 0x6e4; 0x6e7, 0x6e8; 0x6ea, 0x6ed; 0x711, 0x711; 0x730, 0x74a; 0x7a6, 0x7b0; 0x7eb, 0x7f3; 0x7fd, 0x7fd; 0x816, 0x819; 0x81b, 0x823; 0x825, 0x827; 0x829, 0x82d; 0x859, 0x85b; 0x897, 0x89f; 0x8ca, 0x8e1; 0x8e3, 0x902; 0x93a, 0x93a; 0x93c, 0x93c; 0x941, 0x948; 0x94d, 0x94d; 0x951, 0x957; 0x962, 0x963; 0x981, 0x981; 0x9bc, 0x9bc; 0x9c1, 0x9c4; 0x9cd, 0x9cd; 0x9e2, 0x9e3; 0x9fe, 0x9fe; 0xa01, 0xa02; 0xa3c, 0xa3c; 0xa41, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4d; 0xa51, 0xa51; 0xa70, 0xa71; 0xa75, 0xa75; 0xa81, 0xa82; 0xabc, 0xabc; 0xac1, 0xac5; 0xac7, 0xac8; 0xacd, 0xacd; 0xae2, 0xae3; 0xafa, 0xaff; 0xb01, 0xb01; 0xb3c, 0xb3c; 0xb3f, 0xb3f; 0xb41, 0xb44; 0xb4d, 0xb4d; 0xb55, 0xb56; 0xb62, 0xb63; 0xb82, 0xb82; 0xbc0, 0xbc0; 0xbcd, 0xbcd; 0xc00, 0xc00; 0xc04, 0xc04; 0xc3c, 0xc3c; 0xc3e, 0xc40; 0xc46, 0xc48; 0xc4a, 0xc4d; 0xc55, 0xc56; 0xc62, 0xc63; 0xc81, 0xc81; 0xcbc, 0xcbc; 0xcbf, 0xcbf; 0xcc6, 0xcc6; 0xccc, 0xccd; 0xce2, 0xce3; 0xd00, 0xd01; 0xd3b, 0xd3c; 0xd41, 0xd44; 0xd4d, 0xd4d; 0xd62, 0xd63; 0xd81, 0xd81; 0xdca, 0xdca; 0xdd2, 0xdd4; 0xdd6, 0xdd6; 0xe31, 0xe31; 0xe34, 0xe3a; 0xe47, 0xe4e; 0xeb1, 0xeb1; 0xeb4, 0xebc; 0xec8, 0xece; 0xf18, 0xf19; 0xf35, 0xf35; 0xf37, 0xf37; 0xf39, 0xf39; 0xf71, 0xf7e; 0xf80, 0xf84; 0xf86, 0xf87; 0xf8d, 0xf97; 0xf99, 0xfbc; 0xfc6, 0xfc6; 0x102d, 0x1030; 0x1032, 0x1037; 0x1039, 0x103a; 0x103d, 0x103e; 0x1058, 0x1059; 0x105e, 0x1060; 0x1071, 0x1074; 0x1082, 0x1082; 0x1085, 0x1086; 0x108d, 0x108d; 0x109d, 0x109d; 0x135d, 0x135f; 0x1712, 0x1714; 0x1732, 0x1733; 0x1752, 0x1753; 0x1772, 0x1773; 0x17b4, 0x17b5; 0x17b7, 0x17bd; 0x17c6, 0x17c6; 0x17c9, 0x17d3; 0x17dd, 0x17dd; 0x180b, 0x180d; 0x180f, 0x180f; 0x1885, 0x1886; 0x18a9, 0x18a9; 0x1920, 0x1922; 0x1927, 0x1928; 0x1932, 0x1932; 0x1939, 0x193b; 0x1a17, 0x1a18; 0x1a1b, 0x1a1b; 0x1a56, 0x1a56; 0x1a58, 0x1a5e; 0x1a60, 0x1a60; 0x1a62, 0x1a62; 0x1a65, 0x1a6c; 0x1a73, 0x1a7c; 0x1a7f, 0x1a7f; 0x1ab0, 0x1abd; 0x1abf, 0x1add; 0x1ae0, 0x1aeb; 0x1b00, 0x1b03; 0x1b34, 0x1b34; 0x1b36, 0x1b3a; 0x1b3c, 0x1b3c; 0x1b42, 0x1b42; 0x1b6b, 0x1b73; 0x1b80, 0x1b81; 0x1ba2, 0x1ba5; 0x1ba8, 0x1ba9; 0x1bab, 0x1bad; 0x1be6, 0x1be6; 0x1be8, 0x1be9; 0x1bed, 0x1bed; 0x1bef, 0x1bf1; 0x1c2c, 0x1c33; 0x1c36, 0x1c37; 0x1cd0, 0x1cd2; 0x1cd4, 0x1ce0; 0x1ce2, 0x1ce8; 0x1ced, 0x1ced; 0x1cf4, 0x1cf4; 0x1cf8, 0x1cf9; 0x1dc0, 0x1dff; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20f0; 0x2cef, 0x2cf1; 0x2d7f, 0x2d7f; 0x2de0, 0x2dff; 0x302a, 0x302d; 0x3099, 0x309a; 0xa66f, 0xa66f; 0xa674, 0xa67d; 0xa69e, 0xa69f; 0xa6f0, 0xa6f1; 0xa802, 0xa802; 0xa806, 0xa806; 0xa80b, 0xa80b; 0xa825, 0xa826; 0xa82c, 0xa82c; 0xa8c4, 0xa8c5; 0xa8e0, 0xa8f1; 0xa8ff, 0xa8ff; 0xa926, 0xa92d; 0xa947, 0xa951; 0xa980, 0xa982; 0xa9b3, 0xa9b3; 0xa9b6, 0xa9b9; 0xa9bc, 0xa9bd; 0xa9e5, 0xa9e5; 0xaa29, 0xaa2e; 0xaa31, 0xaa32; 0xaa35, 0xaa36; 0xaa43, 0xaa43; 0xaa4c, 0xaa4c; 0xaa7c, 0xaa7c; 0xaab0, 0xaab0; 0xaab2, 0xaab4; 0xaab7, 0xaab8; 0xaabe, 0xaabf; 0xaac1, 0xaac1; 0xaaec, 0xaaed; 0xaaf6, 0xaaf6; 0xabe5, 0xabe5; 0xabe8, 0xabe8; 0xabed, 0xabed; 0xfb1e, 0xfb1e; 0xfe00, 0xfe0f; 0xfe20, 0xfe2f; 0x101fd, 0x101fd; 0x102e0, 0x102e0; 0x10376, 0x1037a; 0x10a01, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a0f; 0x10a38, 0x10a3a; 0x10a3f, 0x10a3f; 0x10ae5, 0x10ae6; 0x10d24, 0x10d27; 0x10d69, 0x10d6d; 0x10eab, 0x10eac; 0x10efa, 0x10eff; 0x10f46, 0x10f50; 0x10f82, 0x10f85; 0x11001, 0x11001; 0x11038, 0x11046; 0x11070, 0x11070; 0x11073, 0x11074; 0x1107f, 0x11081; 0x110b3, 0x110b6; 0x110b9, 0x110ba; 0x110c2, 0x110c2; 0x11100, 0x11102; 0x11127, 0x1112b; 0x1112d, 0x11134; 0x11173, 0x11173; 0x11180, 0x11181; 0x111b6, 0x111be; 0x111c9, 0x111cc; 0x111cf, 0x111cf; 0x1122f, 0x11231; 0x11234, 0x11234; 0x11236, 0x11237; 0x1123e, 0x1123e; 0x11241, 0x11241; 0x112df, 0x112df; 0x112e3, 0x112ea; 0x11300, 0x11301; 0x1133b, 0x1133c; 0x11340, 0x11340; 0x11366, 0x1136c; 0x11370, 0x11374; 0x113bb, 0x113c0; 0x113ce, 0x113ce; 0x113d0, 0x113d0; 0x113d2, 0x113d2; 0x113e1, 0x113e2; 0x11438, 0x1143f; 0x11442, 0x11444; 0x11446, 0x11446; 0x1145e, 0x1145e; 0x114b3, 0x114b8; 0x114ba, 0x114ba; 0x114bf, 0x114c0; 0x114c2, 0x114c3; 0x115b2, 0x115b5; 0x115bc, 0x115bd; 0x115bf, 0x115c0; 0x115dc, 0x115dd; 0x11633, 0x1163a; 0x1163d, 0x1163d; 0x1163f, 0x11640; 0x116ab, 0x116ab; 0x116ad, 0x116ad; 0x116b0, 0x116b5; 0x116b7, 0x116b7; 0x1171d, 0x1171d; 0x1171f, 0x1171f; 0x11722, 0x11725; 0x11727, 0x1172b; 0x1182f, 0x11837; 0x11839, 0x1183a; 0x1193b, 0x1193c; 0x1193e, 0x1193e; 0x11943, 0x11943; 0x119d4, 0x119d7; 0x119da, 0x119db; 0x119e0, 0x119e0; 0x11a01, 0x11a0a; 0x11a33, 0x11a38; 0x11a3b, 0x11a3e; 0x11a47, 0x11a47; 0x11a51, 0x11a56; 0x11a59, 0x11a5b; 0x11a8a, 0x11a96; 0x11a98, 0x11a99; 0x11b60, 0x11b60; 0x11b62, 0x11b64; 0x11b66, 0x11b66; 0x11c30, 0x11c36; 0x11c38, 0x11c3d; 0x11c3f, 0x11c3f; 0x11c92, 0x11ca7; 0x11caa, 0x11cb0; 0x11cb2, 0x11cb3; 0x11cb5, 0x11cb6; 0x11d31, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d45; 0x11d47, 0x11d47; 0x11d90, 0x11d91; 0x11d95, 0x11d95; 0x11d97, 0x11d97; 0x11ef3, 0x11ef4; 0x11f00, 0x11f01; 0x11f36, 0x11f3a; 0x11f40, 0x11f40; 0x11f42, 0x11f42; 0x11f5a, 0x11f5a; 0x13440, 0x13440; 0x13447, 0x13455; 0x1611e, 0x16129; 0x1612d, 0x1612f; 0x16af0, 0x16af4; 0x16b30, 0x16b36; 0x16f4f, 0x16f4f; 0x16f8f, 0x16f92; 0x16fe4, 0x16fe4; 0x1bc9d, 0x1bc9e; 0x1cf00, 0x1cf2d; 0x1cf30, 0x1cf46; 0x1d167, 0x1d169; 0x1d17b, 0x1d182; 0x1d185, 0x1d18b; 0x1d1aa, 0x1d1ad; 0x1d242, 0x1d244; 0x1da00, 0x1da36; 0x1da3b, 0x1da6c; 0x1da75, 0x1da75; 0x1da84, 0x1da84; 0x1da9b, 0x1da9f; 0x1daa1, 0x1daaf; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e08f, 0x1e08f; 0x1e130, 0x1e136; 0x1e2ae, 0x1e2ae; 0x1e2ec, 0x1e2ef; 0x1e4ec, 0x1e4ef; 0x1e5ee, 0x1e5ef; 0x1e6e3, 0x1e6e3; 0x1e6e6, 0x1e6e6; 0x1e6ee, 0x1e6ef; 0x1e6f5, 0x1e6f5; 0x1e8d0, 0x1e8d6; 0x1e944, 0x1e94a; 0xe0100, 0xe01ef] let nd = Cset.of_list [0x30, 0x39; 0x660, 0x669; 0x6f0, 0x6f9; 0x7c0, 0x7c9; 0x966, 0x96f; 0x9e6, 0x9ef; 0xa66, 0xa6f; 0xae6, 0xaef; 0xb66, 0xb6f; 0xbe6, 0xbef; 0xc66, 0xc6f; 0xce6, 0xcef; 0xd66, 0xd6f; 0xde6, 0xdef; 0xe50, 0xe59; 0xed0, 0xed9; 0xf20, 0xf29; 0x1040, 0x1049; 0x1090, 0x1099; 0x17e0, 0x17e9; 0x1810, 0x1819; 0x1946, 0x194f; 0x19d0, 0x19d9; 0x1a80, 0x1a89; 0x1a90, 0x1a99; 0x1b50, 0x1b59; 0x1bb0, 0x1bb9; 0x1c40, 0x1c49; 0x1c50, 0x1c59; 0xa620, 0xa629; 0xa8d0, 0xa8d9; 0xa900, 0xa909; 0xa9d0, 0xa9d9; 0xa9f0, 0xa9f9; 0xaa50, 0xaa59; 0xabf0, 0xabf9; 0xff10, 0xff19; 0x104a0, 0x104a9; 0x10d30, 0x10d39; 0x10d40, 0x10d49; 0x11066, 0x1106f; 0x110f0, 0x110f9; 0x11136, 0x1113f; 0x111d0, 0x111d9; 0x112f0, 0x112f9; 0x11450, 0x11459; 0x114d0, 0x114d9; 0x11650, 0x11659; 0x116c0, 0x116c9; 0x116d0, 0x116e3; 0x11730, 0x11739; 0x118e0, 0x118e9; 0x11950, 0x11959; 0x11bf0, 0x11bf9; 0x11c50, 0x11c59; 0x11d50, 0x11d59; 0x11da0, 0x11da9; 0x11de0, 0x11de9; 0x11f50, 0x11f59; 0x16130, 0x16139; 0x16a60, 0x16a69; 0x16ac0, 0x16ac9; 0x16b50, 0x16b59; 0x16d70, 0x16d79; 0x1ccf0, 0x1ccf9; 0x1d7ce, 0x1d7ff; 0x1e140, 0x1e149; 0x1e2f0, 0x1e2f9; 0x1e4f0, 0x1e4f9; 0x1e5f1, 0x1e5fa; 0x1e950, 0x1e959; 0x1fbf0, 0x1fbf9] let nl = Cset.of_list [0x16ee, 0x16f0; 0x2160, 0x2182; 0x2185, 0x2188; 0x3007, 0x3007; 0x3021, 0x3029; 0x3038, 0x303a; 0xa6e6, 0xa6ef; 0x10140, 0x10174; 0x10341, 0x10341; 0x1034a, 0x1034a; 0x103d1, 0x103d5; 0x12400, 0x1246e; 0x16ff4, 0x16ff6] let no = Cset.of_list [0xb2, 0xb3; 0xb9, 0xb9; 0xbc, 0xbe; 0x9f4, 0x9f9; 0xb72, 0xb77; 0xbf0, 0xbf2; 0xc78, 0xc7e; 0xd58, 0xd5e; 0xd70, 0xd78; 0xf2a, 0xf33; 0x1369, 0x137c; 0x17f0, 0x17f9; 0x19da, 0x19da; 0x2070, 0x2070; 0x2074, 0x2079; 0x2080, 0x2089; 0x2150, 0x215f; 0x2189, 0x2189; 0x2460, 0x249b; 0x24ea, 0x24ff; 0x2776, 0x2793; 0x2cfd, 0x2cfd; 0x3192, 0x3195; 0x3220, 0x3229; 0x3248, 0x324f; 0x3251, 0x325f; 0x3280, 0x3289; 0x32b1, 0x32bf; 0xa830, 0xa835; 0x10107, 0x10133; 0x10175, 0x10178; 0x1018a, 0x1018b; 0x102e1, 0x102fb; 0x10320, 0x10323; 0x10858, 0x1085f; 0x10879, 0x1087f; 0x108a7, 0x108af; 0x108fb, 0x108ff; 0x10916, 0x1091b; 0x109bc, 0x109bd; 0x109c0, 0x109cf; 0x109d2, 0x109ff; 0x10a40, 0x10a48; 0x10a7d, 0x10a7e; 0x10a9d, 0x10a9f; 0x10aeb, 0x10aef; 0x10b58, 0x10b5f; 0x10b78, 0x10b7f; 0x10ba9, 0x10baf; 0x10cfa, 0x10cff; 0x10e60, 0x10e7e; 0x10f1d, 0x10f26; 0x10f51, 0x10f54; 0x10fc5, 0x10fcb; 0x11052, 0x11065; 0x111e1, 0x111f4; 0x1173a, 0x1173b; 0x118ea, 0x118f2; 0x11c5a, 0x11c6c; 0x11fc0, 0x11fd4; 0x16b5b, 0x16b61; 0x16e80, 0x16e96; 0x1d2c0, 0x1d2d3; 0x1d2e0, 0x1d2f3; 0x1d360, 0x1d378; 0x1e8c7, 0x1e8cf; 0x1ec71, 0x1ecab; 0x1ecad, 0x1ecaf; 0x1ecb1, 0x1ecb4; 0x1ed01, 0x1ed2d; 0x1ed2f, 0x1ed3d; 0x1f100, 0x1f10c] let pc = Cset.of_list [0x5f, 0x5f; 0x203f, 0x2040; 0x2054, 0x2054; 0xfe33, 0xfe34; 0xfe4d, 0xfe4f; 0xff3f, 0xff3f] let pd = Cset.of_list [0x2d, 0x2d; 0x58a, 0x58a; 0x5be, 0x5be; 0x1400, 0x1400; 0x1806, 0x1806; 0x2010, 0x2015; 0x2e17, 0x2e17; 0x2e1a, 0x2e1a; 0x2e3a, 0x2e3b; 0x2e40, 0x2e40; 0x2e5d, 0x2e5d; 0x301c, 0x301c; 0x3030, 0x3030; 0x30a0, 0x30a0; 0xfe31, 0xfe32; 0xfe58, 0xfe58; 0xfe63, 0xfe63; 0xff0d, 0xff0d; 0x10d6e, 0x10d6e; 0x10ead, 0x10ead] let pe = Cset.of_list [0x29, 0x29; 0x5d, 0x5d; 0x7d, 0x7d; 0xf3b, 0xf3b; 0xf3d, 0xf3d; 0x169c, 0x169c; 0x2046, 0x2046; 0x207e, 0x207e; 0x208e, 0x208e; 0x2309, 0x2309; 0x230b, 0x230b; 0x232a, 0x232a; 0x2769, 0x2769; 0x276b, 0x276b; 0x276d, 0x276d; 0x276f, 0x276f; 0x2771, 0x2771; 0x2773, 0x2773; 0x2775, 0x2775; 0x27c6, 0x27c6; 0x27e7, 0x27e7; 0x27e9, 0x27e9; 0x27eb, 0x27eb; 0x27ed, 0x27ed; 0x27ef, 0x27ef; 0x2984, 0x2984; 0x2986, 0x2986; 0x2988, 0x2988; 0x298a, 0x298a; 0x298c, 0x298c; 0x298e, 0x298e; 0x2990, 0x2990; 0x2992, 0x2992; 0x2994, 0x2994; 0x2996, 0x2996; 0x2998, 0x2998; 0x29d9, 0x29d9; 0x29db, 0x29db; 0x29fd, 0x29fd; 0x2e23, 0x2e23; 0x2e25, 0x2e25; 0x2e27, 0x2e27; 0x2e29, 0x2e29; 0x2e56, 0x2e56; 0x2e58, 0x2e58; 0x2e5a, 0x2e5a; 0x2e5c, 0x2e5c; 0x3009, 0x3009; 0x300b, 0x300b; 0x300d, 0x300d; 0x300f, 0x300f; 0x3011, 0x3011; 0x3015, 0x3015; 0x3017, 0x3017; 0x3019, 0x3019; 0x301b, 0x301b; 0x301e, 0x301f; 0xfd3e, 0xfd3e; 0xfe18, 0xfe18; 0xfe36, 0xfe36; 0xfe38, 0xfe38; 0xfe3a, 0xfe3a; 0xfe3c, 0xfe3c; 0xfe3e, 0xfe3e; 0xfe40, 0xfe40; 0xfe42, 0xfe42; 0xfe44, 0xfe44; 0xfe48, 0xfe48; 0xfe5a, 0xfe5a; 0xfe5c, 0xfe5c; 0xfe5e, 0xfe5e; 0xff09, 0xff09; 0xff3d, 0xff3d; 0xff5d, 0xff5d; 0xff60, 0xff60; 0xff63, 0xff63] let pf = Cset.of_list [0xbb, 0xbb; 0x2019, 0x2019; 0x201d, 0x201d; 0x203a, 0x203a; 0x2e03, 0x2e03; 0x2e05, 0x2e05; 0x2e0a, 0x2e0a; 0x2e0d, 0x2e0d; 0x2e1d, 0x2e1d; 0x2e21, 0x2e21] let pi = Cset.of_list [0xab, 0xab; 0x2018, 0x2018; 0x201b, 0x201c; 0x201f, 0x201f; 0x2039, 0x2039; 0x2e02, 0x2e02; 0x2e04, 0x2e04; 0x2e09, 0x2e09; 0x2e0c, 0x2e0c; 0x2e1c, 0x2e1c; 0x2e20, 0x2e20] let po = Cset.of_list [0x21, 0x23; 0x25, 0x27; 0x2a, 0x2a; 0x2c, 0x2c; 0x2e, 0x2f; 0x3a, 0x3b; 0x3f, 0x40; 0x5c, 0x5c; 0xa1, 0xa1; 0xa7, 0xa7; 0xb6, 0xb7; 0xbf, 0xbf; 0x37e, 0x37e; 0x387, 0x387; 0x55a, 0x55f; 0x589, 0x589; 0x5c0, 0x5c0; 0x5c3, 0x5c3; 0x5c6, 0x5c6; 0x5f3, 0x5f4; 0x609, 0x60a; 0x60c, 0x60d; 0x61b, 0x61b; 0x61d, 0x61f; 0x66a, 0x66d; 0x6d4, 0x6d4; 0x700, 0x70d; 0x7f7, 0x7f9; 0x830, 0x83e; 0x85e, 0x85e; 0x964, 0x965; 0x970, 0x970; 0x9fd, 0x9fd; 0xa76, 0xa76; 0xaf0, 0xaf0; 0xc77, 0xc77; 0xc84, 0xc84; 0xdf4, 0xdf4; 0xe4f, 0xe4f; 0xe5a, 0xe5b; 0xf04, 0xf12; 0xf14, 0xf14; 0xf85, 0xf85; 0xfd0, 0xfd4; 0xfd9, 0xfda; 0x104a, 0x104f; 0x10fb, 0x10fb; 0x1360, 0x1368; 0x166e, 0x166e; 0x16eb, 0x16ed; 0x1735, 0x1736; 0x17d4, 0x17d6; 0x17d8, 0x17da; 0x1800, 0x1805; 0x1807, 0x180a; 0x1944, 0x1945; 0x1a1e, 0x1a1f; 0x1aa0, 0x1aa6; 0x1aa8, 0x1aad; 0x1b4e, 0x1b4f; 0x1b5a, 0x1b60; 0x1b7d, 0x1b7f; 0x1bfc, 0x1bff; 0x1c3b, 0x1c3f; 0x1c7e, 0x1c7f; 0x1cc0, 0x1cc7; 0x1cd3, 0x1cd3; 0x2016, 0x2017; 0x2020, 0x2027; 0x2030, 0x2038; 0x203b, 0x203e; 0x2041, 0x2043; 0x2047, 0x2051; 0x2053, 0x2053; 0x2055, 0x205e; 0x2cf9, 0x2cfc; 0x2cfe, 0x2cff; 0x2d70, 0x2d70; 0x2e00, 0x2e01; 0x2e06, 0x2e08; 0x2e0b, 0x2e0b; 0x2e0e, 0x2e16; 0x2e18, 0x2e19; 0x2e1b, 0x2e1b; 0x2e1e, 0x2e1f; 0x2e2a, 0x2e2e; 0x2e30, 0x2e39; 0x2e3c, 0x2e3f; 0x2e41, 0x2e41; 0x2e43, 0x2e4f; 0x2e52, 0x2e54; 0x3001, 0x3003; 0x303d, 0x303d; 0x30fb, 0x30fb; 0xa4fe, 0xa4ff; 0xa60d, 0xa60f; 0xa673, 0xa673; 0xa67e, 0xa67e; 0xa6f2, 0xa6f7; 0xa874, 0xa877; 0xa8ce, 0xa8cf; 0xa8f8, 0xa8fa; 0xa8fc, 0xa8fc; 0xa92e, 0xa92f; 0xa95f, 0xa95f; 0xa9c1, 0xa9cd; 0xa9de, 0xa9df; 0xaa5c, 0xaa5f; 0xaade, 0xaadf; 0xaaf0, 0xaaf1; 0xabeb, 0xabeb; 0xfe10, 0xfe16; 0xfe19, 0xfe19; 0xfe30, 0xfe30; 0xfe45, 0xfe46; 0xfe49, 0xfe4c; 0xfe50, 0xfe52; 0xfe54, 0xfe57; 0xfe5f, 0xfe61; 0xfe68, 0xfe68; 0xfe6a, 0xfe6b; 0xff01, 0xff03; 0xff05, 0xff07; 0xff0a, 0xff0a; 0xff0c, 0xff0c; 0xff0e, 0xff0f; 0xff1a, 0xff1b; 0xff1f, 0xff20; 0xff3c, 0xff3c; 0xff61, 0xff61; 0xff64, 0xff65; 0x10100, 0x10102; 0x1039f, 0x1039f; 0x103d0, 0x103d0; 0x1056f, 0x1056f; 0x10857, 0x10857; 0x1091f, 0x1091f; 0x1093f, 0x1093f; 0x10a50, 0x10a58; 0x10a7f, 0x10a7f; 0x10af0, 0x10af6; 0x10b39, 0x10b3f; 0x10b99, 0x10b9c; 0x10ed0, 0x10ed0; 0x10f55, 0x10f59; 0x10f86, 0x10f89; 0x11047, 0x1104d; 0x110bb, 0x110bc; 0x110be, 0x110c1; 0x11140, 0x11143; 0x11174, 0x11175; 0x111c5, 0x111c8; 0x111cd, 0x111cd; 0x111db, 0x111db; 0x111dd, 0x111df; 0x11238, 0x1123d; 0x112a9, 0x112a9; 0x113d4, 0x113d5; 0x113d7, 0x113d8; 0x1144b, 0x1144f; 0x1145a, 0x1145b; 0x1145d, 0x1145d; 0x114c6, 0x114c6; 0x115c1, 0x115d7; 0x11641, 0x11643; 0x11660, 0x1166c; 0x116b9, 0x116b9; 0x1173c, 0x1173e; 0x1183b, 0x1183b; 0x11944, 0x11946; 0x119e2, 0x119e2; 0x11a3f, 0x11a46; 0x11a9a, 0x11a9c; 0x11a9e, 0x11aa2; 0x11b00, 0x11b09; 0x11be1, 0x11be1; 0x11c41, 0x11c45; 0x11c70, 0x11c71; 0x11ef7, 0x11ef8; 0x11f43, 0x11f4f; 0x11fff, 0x11fff; 0x12470, 0x12474; 0x12ff1, 0x12ff2; 0x16a6e, 0x16a6f; 0x16af5, 0x16af5; 0x16b37, 0x16b3b; 0x16b44, 0x16b44; 0x16d6d, 0x16d6f; 0x16e97, 0x16e9a; 0x16fe2, 0x16fe2; 0x1bc9f, 0x1bc9f; 0x1da87, 0x1da8b; 0x1e5ff, 0x1e5ff; 0x1e95e, 0x1e95f] let ps = Cset.of_list [0x28, 0x28; 0x5b, 0x5b; 0x7b, 0x7b; 0xf3a, 0xf3a; 0xf3c, 0xf3c; 0x169b, 0x169b; 0x201a, 0x201a; 0x201e, 0x201e; 0x2045, 0x2045; 0x207d, 0x207d; 0x208d, 0x208d; 0x2308, 0x2308; 0x230a, 0x230a; 0x2329, 0x2329; 0x2768, 0x2768; 0x276a, 0x276a; 0x276c, 0x276c; 0x276e, 0x276e; 0x2770, 0x2770; 0x2772, 0x2772; 0x2774, 0x2774; 0x27c5, 0x27c5; 0x27e6, 0x27e6; 0x27e8, 0x27e8; 0x27ea, 0x27ea; 0x27ec, 0x27ec; 0x27ee, 0x27ee; 0x2983, 0x2983; 0x2985, 0x2985; 0x2987, 0x2987; 0x2989, 0x2989; 0x298b, 0x298b; 0x298d, 0x298d; 0x298f, 0x298f; 0x2991, 0x2991; 0x2993, 0x2993; 0x2995, 0x2995; 0x2997, 0x2997; 0x29d8, 0x29d8; 0x29da, 0x29da; 0x29fc, 0x29fc; 0x2e22, 0x2e22; 0x2e24, 0x2e24; 0x2e26, 0x2e26; 0x2e28, 0x2e28; 0x2e42, 0x2e42; 0x2e55, 0x2e55; 0x2e57, 0x2e57; 0x2e59, 0x2e59; 0x2e5b, 0x2e5b; 0x3008, 0x3008; 0x300a, 0x300a; 0x300c, 0x300c; 0x300e, 0x300e; 0x3010, 0x3010; 0x3014, 0x3014; 0x3016, 0x3016; 0x3018, 0x3018; 0x301a, 0x301a; 0x301d, 0x301d; 0xfd3f, 0xfd3f; 0xfe17, 0xfe17; 0xfe35, 0xfe35; 0xfe37, 0xfe37; 0xfe39, 0xfe39; 0xfe3b, 0xfe3b; 0xfe3d, 0xfe3d; 0xfe3f, 0xfe3f; 0xfe41, 0xfe41; 0xfe43, 0xfe43; 0xfe47, 0xfe47; 0xfe59, 0xfe59; 0xfe5b, 0xfe5b; 0xfe5d, 0xfe5d; 0xff08, 0xff08; 0xff3b, 0xff3b; 0xff5b, 0xff5b; 0xff5f, 0xff5f; 0xff62, 0xff62] let sc = Cset.of_list [0x24, 0x24; 0xa2, 0xa5; 0x58f, 0x58f; 0x60b, 0x60b; 0x7fe, 0x7ff; 0x9f2, 0x9f3; 0x9fb, 0x9fb; 0xaf1, 0xaf1; 0xbf9, 0xbf9; 0xe3f, 0xe3f; 0x17db, 0x17db; 0x20a0, 0x20c1; 0xa838, 0xa838; 0xfdfc, 0xfdfc; 0xfe69, 0xfe69; 0xff04, 0xff04; 0xffe0, 0xffe1; 0xffe5, 0xffe6; 0x11fdd, 0x11fe0; 0x1e2ff, 0x1e2ff; 0x1ecb0, 0x1ecb0] let sk = Cset.of_list [0x5e, 0x5e; 0x60, 0x60; 0xa8, 0xa8; 0xaf, 0xaf; 0xb4, 0xb4; 0xb8, 0xb8; 0x2c2, 0x2c5; 0x2d2, 0x2df; 0x2e5, 0x2eb; 0x2ed, 0x2ed; 0x2ef, 0x2ff; 0x375, 0x375; 0x384, 0x385; 0x888, 0x888; 0x1fbd, 0x1fbd; 0x1fbf, 0x1fc1; 0x1fcd, 0x1fcf; 0x1fdd, 0x1fdf; 0x1fed, 0x1fef; 0x1ffd, 0x1ffe; 0x309b, 0x309c; 0xa700, 0xa716; 0xa720, 0xa721; 0xa789, 0xa78a; 0xab5b, 0xab5b; 0xab6a, 0xab6b; 0xfbb2, 0xfbc2; 0xff3e, 0xff3e; 0xff40, 0xff40; 0xffe3, 0xffe3; 0x1f3fb, 0x1f3ff] let sm = Cset.of_list [0x2b, 0x2b; 0x3c, 0x3e; 0x7c, 0x7c; 0x7e, 0x7e; 0xac, 0xac; 0xb1, 0xb1; 0xd7, 0xd7; 0xf7, 0xf7; 0x3f6, 0x3f6; 0x606, 0x608; 0x2044, 0x2044; 0x2052, 0x2052; 0x207a, 0x207c; 0x208a, 0x208c; 0x2118, 0x2118; 0x2140, 0x2144; 0x214b, 0x214b; 0x2190, 0x2194; 0x219a, 0x219b; 0x21a0, 0x21a0; 0x21a3, 0x21a3; 0x21a6, 0x21a6; 0x21ae, 0x21ae; 0x21ce, 0x21cf; 0x21d2, 0x21d2; 0x21d4, 0x21d4; 0x21f4, 0x22ff; 0x2320, 0x2321; 0x237c, 0x237c; 0x239b, 0x23b3; 0x23dc, 0x23e1; 0x25b7, 0x25b7; 0x25c1, 0x25c1; 0x25f8, 0x25ff; 0x266f, 0x266f; 0x27c0, 0x27c4; 0x27c7, 0x27e5; 0x27f0, 0x27ff; 0x2900, 0x2982; 0x2999, 0x29d7; 0x29dc, 0x29fb; 0x29fe, 0x2aff; 0x2b30, 0x2b44; 0x2b47, 0x2b4c; 0xfb29, 0xfb29; 0xfe62, 0xfe62; 0xfe64, 0xfe66; 0xff0b, 0xff0b; 0xff1c, 0xff1e; 0xff5c, 0xff5c; 0xff5e, 0xff5e; 0xffe2, 0xffe2; 0xffe9, 0xffec; 0x10d8e, 0x10d8f; 0x1cef0, 0x1cef0; 0x1d6c1, 0x1d6c1; 0x1d6db, 0x1d6db; 0x1d6fb, 0x1d6fb; 0x1d715, 0x1d715; 0x1d735, 0x1d735; 0x1d74f, 0x1d74f; 0x1d76f, 0x1d76f; 0x1d789, 0x1d789; 0x1d7a9, 0x1d7a9; 0x1d7c3, 0x1d7c3; 0x1eef0, 0x1eef1; 0x1f8d0, 0x1f8d8] let so = Cset.of_list [0xa6, 0xa6; 0xa9, 0xa9; 0xae, 0xae; 0xb0, 0xb0; 0x482, 0x482; 0x58d, 0x58e; 0x60e, 0x60f; 0x6de, 0x6de; 0x6e9, 0x6e9; 0x6fd, 0x6fe; 0x7f6, 0x7f6; 0x9fa, 0x9fa; 0xb70, 0xb70; 0xbf3, 0xbf8; 0xbfa, 0xbfa; 0xc7f, 0xc7f; 0xd4f, 0xd4f; 0xd79, 0xd79; 0xf01, 0xf03; 0xf13, 0xf13; 0xf15, 0xf17; 0xf1a, 0xf1f; 0xf34, 0xf34; 0xf36, 0xf36; 0xf38, 0xf38; 0xfbe, 0xfc5; 0xfc7, 0xfcc; 0xfce, 0xfcf; 0xfd5, 0xfd8; 0x109e, 0x109f; 0x1390, 0x1399; 0x166d, 0x166d; 0x1940, 0x1940; 0x19de, 0x19ff; 0x1b61, 0x1b6a; 0x1b74, 0x1b7c; 0x2100, 0x2101; 0x2103, 0x2106; 0x2108, 0x2109; 0x2114, 0x2114; 0x2116, 0x2117; 0x211e, 0x2123; 0x2125, 0x2125; 0x2127, 0x2127; 0x2129, 0x2129; 0x212e, 0x212e; 0x213a, 0x213b; 0x214a, 0x214a; 0x214c, 0x214d; 0x214f, 0x214f; 0x218a, 0x218b; 0x2195, 0x2199; 0x219c, 0x219f; 0x21a1, 0x21a2; 0x21a4, 0x21a5; 0x21a7, 0x21ad; 0x21af, 0x21cd; 0x21d0, 0x21d1; 0x21d3, 0x21d3; 0x21d5, 0x21f3; 0x2300, 0x2307; 0x230c, 0x231f; 0x2322, 0x2328; 0x232b, 0x237b; 0x237d, 0x239a; 0x23b4, 0x23db; 0x23e2, 0x2429; 0x2440, 0x244a; 0x249c, 0x24e9; 0x2500, 0x25b6; 0x25b8, 0x25c0; 0x25c2, 0x25f7; 0x2600, 0x266e; 0x2670, 0x2767; 0x2794, 0x27bf; 0x2800, 0x28ff; 0x2b00, 0x2b2f; 0x2b45, 0x2b46; 0x2b4d, 0x2b73; 0x2b76, 0x2bff; 0x2ce5, 0x2cea; 0x2e50, 0x2e51; 0x2e80, 0x2e99; 0x2e9b, 0x2ef3; 0x2f00, 0x2fd5; 0x2ff0, 0x2fff; 0x3004, 0x3004; 0x3012, 0x3013; 0x3020, 0x3020; 0x3036, 0x3037; 0x303e, 0x303f; 0x3190, 0x3191; 0x3196, 0x319f; 0x31c0, 0x31e5; 0x31ef, 0x31ef; 0x3200, 0x321e; 0x322a, 0x3247; 0x3250, 0x3250; 0x3260, 0x327f; 0x328a, 0x32b0; 0x32c0, 0x33ff; 0x4dc0, 0x4dff; 0xa490, 0xa4c6; 0xa828, 0xa82b; 0xa836, 0xa837; 0xa839, 0xa839; 0xaa77, 0xaa79; 0xfbc3, 0xfbd2; 0xfd40, 0xfd4f; 0xfd90, 0xfd91; 0xfdc8, 0xfdcf; 0xfdfd, 0xfdff; 0xffe4, 0xffe4; 0xffe8, 0xffe8; 0xffed, 0xffee; 0xfffc, 0xfffd; 0x10137, 0x1013f; 0x10179, 0x10189; 0x1018c, 0x1018e; 0x10190, 0x1019c; 0x101a0, 0x101a0; 0x101d0, 0x101fc; 0x10877, 0x10878; 0x10ac8, 0x10ac8; 0x10ed1, 0x10ed8; 0x1173f, 0x1173f; 0x11fd5, 0x11fdc; 0x11fe1, 0x11ff1; 0x16b3c, 0x16b3f; 0x16b45, 0x16b45; 0x1bc9c, 0x1bc9c; 0x1cc00, 0x1ccef; 0x1ccfa, 0x1ccfc; 0x1cd00, 0x1ceb3; 0x1ceba, 0x1ced0; 0x1cee0, 0x1ceef; 0x1cf50, 0x1cfc3; 0x1d000, 0x1d0f5; 0x1d100, 0x1d126; 0x1d129, 0x1d164; 0x1d16a, 0x1d16c; 0x1d183, 0x1d184; 0x1d18c, 0x1d1a9; 0x1d1ae, 0x1d1ea; 0x1d200, 0x1d241; 0x1d245, 0x1d245; 0x1d300, 0x1d356; 0x1d800, 0x1d9ff; 0x1da37, 0x1da3a; 0x1da6d, 0x1da74; 0x1da76, 0x1da83; 0x1da85, 0x1da86; 0x1e14f, 0x1e14f; 0x1ecac, 0x1ecac; 0x1ed2e, 0x1ed2e; 0x1f000, 0x1f02b; 0x1f030, 0x1f093; 0x1f0a0, 0x1f0ae; 0x1f0b1, 0x1f0bf; 0x1f0c1, 0x1f0cf; 0x1f0d1, 0x1f0f5; 0x1f10d, 0x1f1ad; 0x1f1e6, 0x1f202; 0x1f210, 0x1f23b; 0x1f240, 0x1f248; 0x1f250, 0x1f251; 0x1f260, 0x1f265; 0x1f300, 0x1f3fa; 0x1f400, 0x1f6d8; 0x1f6dc, 0x1f6ec; 0x1f6f0, 0x1f6fc; 0x1f700, 0x1f7d9; 0x1f7e0, 0x1f7eb; 0x1f7f0, 0x1f7f0; 0x1f800, 0x1f80b; 0x1f810, 0x1f847; 0x1f850, 0x1f859; 0x1f860, 0x1f887; 0x1f890, 0x1f8ad; 0x1f8b0, 0x1f8bb; 0x1f8c0, 0x1f8c1; 0x1f900, 0x1fa57; 0x1fa60, 0x1fa6d; 0x1fa70, 0x1fa7c; 0x1fa80, 0x1fa8a; 0x1fa8e, 0x1fac6; 0x1fac8, 0x1fac8; 0x1facd, 0x1fadc; 0x1fadf, 0x1faea; 0x1faef, 0x1faf8; 0x1fb00, 0x1fb92; 0x1fb94, 0x1fbef; 0x1fbfa, 0x1fbfa] let zl = Cset.of_list [0x2028, 0x2028] let zp = Cset.of_list [0x2029, 0x2029] let zs = Cset.of_list [0x20, 0x20; 0xa0, 0xa0; 0x1680, 0x1680; 0x2000, 0x200a; 0x202f, 0x202f; 0x205f, 0x205f; 0x3000, 0x3000] let list = [ ("cc", cc); ("cf", cf); ("cn", cn); ("co", co); ("cs", cs); ("ll", ll); ("lm", lm); ("lo", lo); ("lt", lt); ("lu", lu); ("mc", mc); ("me", me); ("mn", mn); ("nd", nd); ("nl", nl); ("no", no); ("pc", pc); ("pd", pd); ("pe", pe); ("pf", pf); ("pi", pi); ("po", po); ("ps", ps); ("sc", sc); ("sk", sk); ("sm", sm); ("so", so); ("zl", zl); ("zp", zp); ("zs", zs) ] end module Properties = struct let alphabetic = Cset.of_list [0x41, 0x5a; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x345, 0x345; 0x363, 0x374; 0x376, 0x377; 0x37a, 0x37d; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x48a, 0x52f; 0x531, 0x556; 0x559, 0x559; 0x560, 0x588; 0x5b0, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c7; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x610, 0x61a; 0x620, 0x657; 0x659, 0x65f; 0x66e, 0x6d3; 0x6d5, 0x6dc; 0x6e1, 0x6e8; 0x6ed, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x73f; 0x74d, 0x7b1; 0x7ca, 0x7ea; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x800, 0x817; 0x81a, 0x82c; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x897, 0x897; 0x8a0, 0x8c9; 0x8d4, 0x8df; 0x8e3, 0x8e9; 0x8f0, 0x93b; 0x93d, 0x94c; 0x94e, 0x950; 0x955, 0x963; 0x971, 0x983; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9cc; 0x9ce, 0x9ce; 0x9d7, 0x9d7; 0x9dc, 0x9dd; 0x9df, 0x9e3; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa01, 0xa03; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4c; 0xa51, 0xa51; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa70, 0xa75; 0xa81, 0xa83; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xac5; 0xac7, 0xac9; 0xacb, 0xacc; 0xad0, 0xad0; 0xae0, 0xae3; 0xaf9, 0xafc; 0xb01, 0xb03; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4c; 0xb56, 0xb57; 0xb5c, 0xb5d; 0xb5f, 0xb63; 0xb71, 0xb71; 0xb82, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcc; 0xbd0, 0xbd0; 0xbd7, 0xbd7; 0xc00, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4c; 0xc55, 0xc56; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc63; 0xc80, 0xc83; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccc; 0xcd5, 0xcd6; 0xcdc, 0xcde; 0xce0, 0xce3; 0xcf1, 0xcf3; 0xd00, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4c; 0xd4e, 0xd4e; 0xd54, 0xd57; 0xd5f, 0xd63; 0xd7a, 0xd7f; 0xd81, 0xd83; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xdf2, 0xdf3; 0xe01, 0xe3a; 0xe40, 0xe46; 0xe4d, 0xe4d; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb9; 0xebb, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xecd, 0xecd; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf71, 0xf83; 0xf88, 0xf97; 0xf99, 0xfbc; 0x1000, 0x1036; 0x1038, 0x1038; 0x103b, 0x103f; 0x1050, 0x108f; 0x109a, 0x109d; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1713; 0x171f, 0x1733; 0x1740, 0x1753; 0x1760, 0x176c; 0x176e, 0x1770; 0x1772, 0x1773; 0x1780, 0x17b3; 0x17b6, 0x17c8; 0x17d7, 0x17d7; 0x17dc, 0x17dc; 0x1820, 0x1878; 0x1880, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1920, 0x192b; 0x1930, 0x1938; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a1b; 0x1a20, 0x1a5e; 0x1a61, 0x1a74; 0x1aa7, 0x1aa7; 0x1abf, 0x1ac0; 0x1acc, 0x1ace; 0x1b00, 0x1b33; 0x1b35, 0x1b43; 0x1b45, 0x1b4c; 0x1b80, 0x1ba9; 0x1bac, 0x1baf; 0x1bba, 0x1be5; 0x1be7, 0x1bf1; 0x1c00, 0x1c36; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x1d00, 0x1dbf; 0x1dd3, 0x1df4; 0x1e00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x212d; 0x212f, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x24b6, 0x24e9; 0x2c00, 0x2ce4; 0x2ceb, 0x2cee; 0x2cf2, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x2de0, 0x2dff; 0x2e2f, 0x2e2f; 0x3005, 0x3007; 0x3021, 0x3029; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x309d, 0x309f; 0x30a1, 0x30fa; 0x30fc, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa640, 0xa66e; 0xa674, 0xa67b; 0xa67f, 0xa6ef; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dc; 0xa7f1, 0xa805; 0xa807, 0xa827; 0xa840, 0xa873; 0xa880, 0xa8c3; 0xa8c5, 0xa8c5; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8ff; 0xa90a, 0xa92a; 0xa930, 0xa952; 0xa960, 0xa97c; 0xa980, 0xa9b2; 0xa9b4, 0xa9bf; 0xa9cf, 0xa9cf; 0xa9e0, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa36; 0xaa40, 0xaa4d; 0xaa60, 0xaa76; 0xaa7a, 0xaabe; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaef; 0xaaf2, 0xaaf5; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabea; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff21, 0xff3a; 0xff41, 0xff5a; 0xff66, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x1037a; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d27; 0x10d4a, 0x10d65; 0x10d69, 0x10d69; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eab, 0x10eac; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10efa, 0x10efc; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11000, 0x11045; 0x11071, 0x11075; 0x11080, 0x110b8; 0x110c2, 0x110c2; 0x110d0, 0x110e8; 0x11100, 0x11132; 0x11144, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11180, 0x111bf; 0x111c1, 0x111c4; 0x111ce, 0x111cf; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x11234; 0x11237, 0x11237; 0x1123e, 0x11241; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112e8; 0x11300, 0x11303; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134c; 0x11350, 0x11350; 0x11357, 0x11357; 0x1135d, 0x11363; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113cd; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11441; 0x11443, 0x11445; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114c1; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115b5; 0x115b8, 0x115be; 0x115d8, 0x115dd; 0x11600, 0x1163e; 0x11640, 0x11640; 0x11644, 0x11644; 0x11680, 0x116b5; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x1171d, 0x1172a; 0x11740, 0x11746; 0x11800, 0x11838; 0x118a0, 0x118df; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x11935; 0x11937, 0x11938; 0x1193b, 0x1193c; 0x1193f, 0x11942; 0x119a0, 0x119a7; 0x119aa, 0x119d7; 0x119da, 0x119df; 0x119e1, 0x119e1; 0x119e3, 0x119e4; 0x11a00, 0x11a32; 0x11a35, 0x11a3e; 0x11a50, 0x11a97; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b60, 0x11b67; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c36; 0x11c38, 0x11c3e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d41; 0x11d43, 0x11d43; 0x11d46, 0x11d47; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d96; 0x11d98, 0x11d98; 0x11db0, 0x11ddb; 0x11ee0, 0x11ef6; 0x11f00, 0x11f10; 0x11f12, 0x11f3a; 0x11f3e, 0x11f40; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246e; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1612e; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b40, 0x16b43; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f4f, 0x16f87; 0x16f8f, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff0, 0x16ff6; 0x17000, 0x18cd5; 0x18cff, 0x18d1e; 0x18d80, 0x18df2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b122; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b167; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1bc9e, 0x1bc9e; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1df00, 0x1df1e; 0x1df25, 0x1df2a; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e030, 0x1e06d; 0x1e08f, 0x1e08f; 0x1e100, 0x1e12c; 0x1e137, 0x1e13d; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4eb; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6f5; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e900, 0x1e943; 0x1e947, 0x1e947; 0x1e94b, 0x1e94b; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189; 0x20000, 0x2a6df; 0x2a700, 0x2b81d; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479] let ascii_hex_digit = Cset.of_list [0x30, 0x39; 0x41, 0x46; 0x61, 0x66] let hex_digit = Cset.of_list [0x30, 0x39; 0x41, 0x46; 0x61, 0x66; 0xff10, 0xff19; 0xff21, 0xff26; 0xff41, 0xff46] let id_continue = Cset.of_list [0x30, 0x39; 0x41, 0x5a; 0x5f, 0x5f; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xb7, 0xb7; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x300, 0x374; 0x376, 0x377; 0x37a, 0x37d; 0x37f, 0x37f; 0x386, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x483, 0x487; 0x48a, 0x52f; 0x531, 0x556; 0x559, 0x559; 0x560, 0x588; 0x591, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c7; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x610, 0x61a; 0x620, 0x669; 0x66e, 0x6d3; 0x6d5, 0x6dc; 0x6df, 0x6e8; 0x6ea, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x74a; 0x74d, 0x7b1; 0x7c0, 0x7f5; 0x7fa, 0x7fa; 0x7fd, 0x7fd; 0x800, 0x82d; 0x840, 0x85b; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x897, 0x8e1; 0x8e3, 0x963; 0x966, 0x96f; 0x971, 0x983; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bc, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9ce; 0x9d7, 0x9d7; 0x9dc, 0x9dd; 0x9df, 0x9e3; 0x9e6, 0x9f1; 0x9fc, 0x9fc; 0x9fe, 0x9fe; 0xa01, 0xa03; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa3c, 0xa3c; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4d; 0xa51, 0xa51; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa66, 0xa75; 0xa81, 0xa83; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabc, 0xac5; 0xac7, 0xac9; 0xacb, 0xacd; 0xad0, 0xad0; 0xae0, 0xae3; 0xae6, 0xaef; 0xaf9, 0xaff; 0xb01, 0xb03; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3c, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4d; 0xb55, 0xb57; 0xb5c, 0xb5d; 0xb5f, 0xb63; 0xb66, 0xb6f; 0xb71, 0xb71; 0xb82, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcd; 0xbd0, 0xbd0; 0xbd7, 0xbd7; 0xbe6, 0xbef; 0xc00, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3c, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4d; 0xc55, 0xc56; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc63; 0xc66, 0xc6f; 0xc80, 0xc83; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbc, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccd; 0xcd5, 0xcd6; 0xcdc, 0xcde; 0xce0, 0xce3; 0xce6, 0xcef; 0xcf1, 0xcf3; 0xd00, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4e; 0xd54, 0xd57; 0xd5f, 0xd63; 0xd66, 0xd6f; 0xd7a, 0xd7f; 0xd81, 0xd83; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xdca, 0xdca; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xde6, 0xdef; 0xdf2, 0xdf3; 0xe01, 0xe3a; 0xe40, 0xe4e; 0xe50, 0xe59; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xec8, 0xece; 0xed0, 0xed9; 0xedc, 0xedf; 0xf00, 0xf00; 0xf18, 0xf19; 0xf20, 0xf29; 0xf35, 0xf35; 0xf37, 0xf37; 0xf39, 0xf39; 0xf3e, 0xf47; 0xf49, 0xf6c; 0xf71, 0xf84; 0xf86, 0xf97; 0xf99, 0xfbc; 0xfc6, 0xfc6; 0x1000, 0x1049; 0x1050, 0x109d; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x135d, 0x135f; 0x1369, 0x1371; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1715; 0x171f, 0x1734; 0x1740, 0x1753; 0x1760, 0x176c; 0x176e, 0x1770; 0x1772, 0x1773; 0x1780, 0x17d3; 0x17d7, 0x17d7; 0x17dc, 0x17dd; 0x17e0, 0x17e9; 0x180b, 0x180d; 0x180f, 0x1819; 0x1820, 0x1878; 0x1880, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1920, 0x192b; 0x1930, 0x193b; 0x1946, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x19d0, 0x19da; 0x1a00, 0x1a1b; 0x1a20, 0x1a5e; 0x1a60, 0x1a7c; 0x1a7f, 0x1a89; 0x1a90, 0x1a99; 0x1aa7, 0x1aa7; 0x1ab0, 0x1abd; 0x1abf, 0x1add; 0x1ae0, 0x1aeb; 0x1b00, 0x1b4c; 0x1b50, 0x1b59; 0x1b6b, 0x1b73; 0x1b80, 0x1bf3; 0x1c00, 0x1c37; 0x1c40, 0x1c49; 0x1c4d, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1cd0, 0x1cd2; 0x1cd4, 0x1cfa; 0x1d00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x200c, 0x200d; 0x203f, 0x2040; 0x2054, 0x2054; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20f0; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d7f, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x2de0, 0x2dff; 0x3005, 0x3007; 0x3021, 0x302f; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x3099, 0x309f; 0x30a1, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa62b; 0xa640, 0xa66f; 0xa674, 0xa67d; 0xa67f, 0xa6f1; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dc; 0xa7f1, 0xa827; 0xa82c, 0xa82c; 0xa840, 0xa873; 0xa880, 0xa8c5; 0xa8d0, 0xa8d9; 0xa8e0, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa92d; 0xa930, 0xa953; 0xa960, 0xa97c; 0xa980, 0xa9c0; 0xa9cf, 0xa9d9; 0xa9e0, 0xa9fe; 0xaa00, 0xaa36; 0xaa40, 0xaa4d; 0xaa50, 0xaa59; 0xaa60, 0xaa76; 0xaa7a, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaef; 0xaaf2, 0xaaf6; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabea; 0xabec, 0xabed; 0xabf0, 0xabf9; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe00, 0xfe0f; 0xfe20, 0xfe2f; 0xfe33, 0xfe34; 0xfe4d, 0xfe4f; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff10, 0xff19; 0xff21, 0xff3a; 0xff3f, 0xff3f; 0xff41, 0xff5a; 0xff65, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x101fd, 0x101fd; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x102e0, 0x102e0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x1037a; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104a0, 0x104a9; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a38, 0x10a3a; 0x10a3f, 0x10a3f; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae6; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d27; 0x10d30, 0x10d39; 0x10d40, 0x10d65; 0x10d69, 0x10d6d; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eab, 0x10eac; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10efa, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f50; 0x10f70, 0x10f85; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11000, 0x11046; 0x11066, 0x11075; 0x1107f, 0x110ba; 0x110c2, 0x110c2; 0x110d0, 0x110e8; 0x110f0, 0x110f9; 0x11100, 0x11134; 0x11136, 0x1113f; 0x11144, 0x11147; 0x11150, 0x11173; 0x11176, 0x11176; 0x11180, 0x111c4; 0x111c9, 0x111cc; 0x111ce, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x11237; 0x1123e, 0x11241; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112ea; 0x112f0, 0x112f9; 0x11300, 0x11303; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133b, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134d; 0x11350, 0x11350; 0x11357, 0x11357; 0x1135d, 0x11363; 0x11366, 0x1136c; 0x11370, 0x11374; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113d3; 0x113e1, 0x113e2; 0x11400, 0x1144a; 0x11450, 0x11459; 0x1145e, 0x11461; 0x11480, 0x114c5; 0x114c7, 0x114c7; 0x114d0, 0x114d9; 0x11580, 0x115b5; 0x115b8, 0x115c0; 0x115d8, 0x115dd; 0x11600, 0x11640; 0x11644, 0x11644; 0x11650, 0x11659; 0x11680, 0x116b8; 0x116c0, 0x116c9; 0x116d0, 0x116e3; 0x11700, 0x1171a; 0x1171d, 0x1172b; 0x11730, 0x11739; 0x11740, 0x11746; 0x11800, 0x1183a; 0x118a0, 0x118e9; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x11935; 0x11937, 0x11938; 0x1193b, 0x11943; 0x11950, 0x11959; 0x119a0, 0x119a7; 0x119aa, 0x119d7; 0x119da, 0x119e1; 0x119e3, 0x119e4; 0x11a00, 0x11a3e; 0x11a47, 0x11a47; 0x11a50, 0x11a99; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b60, 0x11b67; 0x11bc0, 0x11be0; 0x11bf0, 0x11bf9; 0x11c00, 0x11c08; 0x11c0a, 0x11c36; 0x11c38, 0x11c40; 0x11c50, 0x11c59; 0x11c72, 0x11c8f; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d47; 0x11d50, 0x11d59; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d98; 0x11da0, 0x11da9; 0x11db0, 0x11ddb; 0x11de0, 0x11de9; 0x11ee0, 0x11ef6; 0x11f00, 0x11f10; 0x11f12, 0x11f3a; 0x11f3e, 0x11f42; 0x11f50, 0x11f5a; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246e; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13440, 0x13455; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x16139; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a60, 0x16a69; 0x16a70, 0x16abe; 0x16ac0, 0x16ac9; 0x16ad0, 0x16aed; 0x16af0, 0x16af4; 0x16b00, 0x16b36; 0x16b40, 0x16b43; 0x16b50, 0x16b59; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16d70, 0x16d79; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f4f, 0x16f87; 0x16f8f, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe4; 0x16ff0, 0x16ff6; 0x17000, 0x18cd5; 0x18cff, 0x18d1e; 0x18d80, 0x18df2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b122; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b167; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1bc9d, 0x1bc9e; 0x1ccf0, 0x1ccf9; 0x1cf00, 0x1cf2d; 0x1cf30, 0x1cf46; 0x1d165, 0x1d169; 0x1d16d, 0x1d172; 0x1d17b, 0x1d182; 0x1d185, 0x1d18b; 0x1d1aa, 0x1d1ad; 0x1d242, 0x1d244; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1da00, 0x1da36; 0x1da3b, 0x1da6c; 0x1da75, 0x1da75; 0x1da84, 0x1da84; 0x1da9b, 0x1da9f; 0x1daa1, 0x1daaf; 0x1df00, 0x1df1e; 0x1df25, 0x1df2a; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e030, 0x1e06d; 0x1e08f, 0x1e08f; 0x1e100, 0x1e12c; 0x1e130, 0x1e13d; 0x1e140, 0x1e149; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ae; 0x1e2c0, 0x1e2f9; 0x1e4d0, 0x1e4f9; 0x1e5d0, 0x1e5fa; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6f5; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e8d0, 0x1e8d6; 0x1e900, 0x1e94b; 0x1e950, 0x1e959; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1fbf0, 0x1fbf9; 0x20000, 0x2a6df; 0x2a700, 0x2b81d; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0xe0100, 0xe01ef] let id_start = Cset.of_list [0x41, 0x5a; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x370, 0x374; 0x376, 0x377; 0x37a, 0x37d; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x48a, 0x52f; 0x531, 0x556; 0x559, 0x559; 0x560, 0x588; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x620, 0x64a; 0x66e, 0x66f; 0x671, 0x6d3; 0x6d5, 0x6d5; 0x6e5, 0x6e6; 0x6ee, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x710; 0x712, 0x72f; 0x74d, 0x7a5; 0x7b1, 0x7b1; 0x7ca, 0x7ea; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x800, 0x815; 0x81a, 0x81a; 0x824, 0x824; 0x828, 0x828; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x8a0, 0x8c9; 0x904, 0x939; 0x93d, 0x93d; 0x950, 0x950; 0x958, 0x961; 0x971, 0x980; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9bd; 0x9ce, 0x9ce; 0x9dc, 0x9dd; 0x9df, 0x9e1; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa72, 0xa74; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xabd; 0xad0, 0xad0; 0xae0, 0xae1; 0xaf9, 0xaf9; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb3d; 0xb5c, 0xb5d; 0xb5f, 0xb61; 0xb71, 0xb71; 0xb83, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbd0, 0xbd0; 0xc05, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc3d; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc61; 0xc80, 0xc80; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcbd; 0xcdc, 0xcde; 0xce0, 0xce1; 0xcf1, 0xcf2; 0xd04, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd3d; 0xd4e, 0xd4e; 0xd54, 0xd56; 0xd5f, 0xd61; 0xd7a, 0xd7f; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xe01, 0xe30; 0xe32, 0xe33; 0xe40, 0xe46; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb0; 0xeb2, 0xeb3; 0xebd, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf88, 0xf8c; 0x1000, 0x102a; 0x103f, 0x103f; 0x1050, 0x1055; 0x105a, 0x105d; 0x1061, 0x1061; 0x1065, 0x1066; 0x106e, 0x1070; 0x1075, 0x1081; 0x108e, 0x108e; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1711; 0x171f, 0x1731; 0x1740, 0x1751; 0x1760, 0x176c; 0x176e, 0x1770; 0x1780, 0x17b3; 0x17d7, 0x17d7; 0x17dc, 0x17dc; 0x1820, 0x1878; 0x1880, 0x18a8; 0x18aa, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a16; 0x1a20, 0x1a54; 0x1aa7, 0x1aa7; 0x1b05, 0x1b33; 0x1b45, 0x1b4c; 0x1b83, 0x1ba0; 0x1bae, 0x1baf; 0x1bba, 0x1be5; 0x1c00, 0x1c23; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x1d00, 0x1dbf; 0x1e00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cee; 0x2cf2, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x3005, 0x3007; 0x3021, 0x3029; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x309b, 0x309f; 0x30a1, 0x30fa; 0x30fc, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa640, 0xa66e; 0xa67f, 0xa69d; 0xa6a0, 0xa6ef; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dc; 0xa7f1, 0xa801; 0xa803, 0xa805; 0xa807, 0xa80a; 0xa80c, 0xa822; 0xa840, 0xa873; 0xa882, 0xa8b3; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8fe; 0xa90a, 0xa925; 0xa930, 0xa946; 0xa960, 0xa97c; 0xa984, 0xa9b2; 0xa9cf, 0xa9cf; 0xa9e0, 0xa9e4; 0xa9e6, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa28; 0xaa40, 0xaa42; 0xaa44, 0xaa4b; 0xaa60, 0xaa76; 0xaa7a, 0xaa7a; 0xaa7e, 0xaaaf; 0xaab1, 0xaab1; 0xaab5, 0xaab6; 0xaab9, 0xaabd; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaea; 0xaaf2, 0xaaf4; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabe2; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb1d; 0xfb1f, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff21, 0xff3a; 0xff41, 0xff5a; 0xff66, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x10375; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a00; 0x10a10, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d23; 0x10d4a, 0x10d65; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11003, 0x11037; 0x11071, 0x11072; 0x11075, 0x11075; 0x11083, 0x110af; 0x110d0, 0x110e8; 0x11103, 0x11126; 0x11144, 0x11144; 0x11147, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11183, 0x111b2; 0x111c1, 0x111c4; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x1122b; 0x1123f, 0x11240; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112de; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x1133d; 0x11350, 0x11350; 0x1135d, 0x11361; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113b7; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11434; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114af; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115ae; 0x115d8, 0x115db; 0x11600, 0x1162f; 0x11644, 0x11644; 0x11680, 0x116aa; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x11740, 0x11746; 0x11800, 0x1182b; 0x118a0, 0x118df; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x1192f; 0x1193f, 0x1193f; 0x11941, 0x11941; 0x119a0, 0x119a7; 0x119aa, 0x119d0; 0x119e1, 0x119e1; 0x119e3, 0x119e3; 0x11a00, 0x11a00; 0x11a0b, 0x11a32; 0x11a3a, 0x11a3a; 0x11a50, 0x11a50; 0x11a5c, 0x11a89; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c2e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d30; 0x11d46, 0x11d46; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d89; 0x11d98, 0x11d98; 0x11db0, 0x11ddb; 0x11ee0, 0x11ef2; 0x11f02, 0x11f02; 0x11f04, 0x11f10; 0x11f12, 0x11f33; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246e; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1611d; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b40, 0x16b43; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f50, 0x16f50; 0x16f93, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff2, 0x16ff6; 0x17000, 0x18cd5; 0x18cff, 0x18d1e; 0x18d80, 0x18df2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b122; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b167; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1df00, 0x1df1e; 0x1df25, 0x1df2a; 0x1e030, 0x1e06d; 0x1e100, 0x1e12c; 0x1e137, 0x1e13d; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4eb; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6e2; 0x1e6e4, 0x1e6e5; 0x1e6e7, 0x1e6ed; 0x1e6f0, 0x1e6f4; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e900, 0x1e943; 0x1e94b, 0x1e94b; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x20000, 0x2a6df; 0x2a700, 0x2b81d; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479] let lowercase = Cset.of_list [0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xdf, 0xf6; 0xf8, 0xff; 0x101, 0x101; 0x103, 0x103; 0x105, 0x105; 0x107, 0x107; 0x109, 0x109; 0x10b, 0x10b; 0x10d, 0x10d; 0x10f, 0x10f; 0x111, 0x111; 0x113, 0x113; 0x115, 0x115; 0x117, 0x117; 0x119, 0x119; 0x11b, 0x11b; 0x11d, 0x11d; 0x11f, 0x11f; 0x121, 0x121; 0x123, 0x123; 0x125, 0x125; 0x127, 0x127; 0x129, 0x129; 0x12b, 0x12b; 0x12d, 0x12d; 0x12f, 0x12f; 0x131, 0x131; 0x133, 0x133; 0x135, 0x135; 0x137, 0x138; 0x13a, 0x13a; 0x13c, 0x13c; 0x13e, 0x13e; 0x140, 0x140; 0x142, 0x142; 0x144, 0x144; 0x146, 0x146; 0x148, 0x149; 0x14b, 0x14b; 0x14d, 0x14d; 0x14f, 0x14f; 0x151, 0x151; 0x153, 0x153; 0x155, 0x155; 0x157, 0x157; 0x159, 0x159; 0x15b, 0x15b; 0x15d, 0x15d; 0x15f, 0x15f; 0x161, 0x161; 0x163, 0x163; 0x165, 0x165; 0x167, 0x167; 0x169, 0x169; 0x16b, 0x16b; 0x16d, 0x16d; 0x16f, 0x16f; 0x171, 0x171; 0x173, 0x173; 0x175, 0x175; 0x177, 0x177; 0x17a, 0x17a; 0x17c, 0x17c; 0x17e, 0x180; 0x183, 0x183; 0x185, 0x185; 0x188, 0x188; 0x18c, 0x18d; 0x192, 0x192; 0x195, 0x195; 0x199, 0x19b; 0x19e, 0x19e; 0x1a1, 0x1a1; 0x1a3, 0x1a3; 0x1a5, 0x1a5; 0x1a8, 0x1a8; 0x1aa, 0x1ab; 0x1ad, 0x1ad; 0x1b0, 0x1b0; 0x1b4, 0x1b4; 0x1b6, 0x1b6; 0x1b9, 0x1ba; 0x1bd, 0x1bf; 0x1c6, 0x1c6; 0x1c9, 0x1c9; 0x1cc, 0x1cc; 0x1ce, 0x1ce; 0x1d0, 0x1d0; 0x1d2, 0x1d2; 0x1d4, 0x1d4; 0x1d6, 0x1d6; 0x1d8, 0x1d8; 0x1da, 0x1da; 0x1dc, 0x1dd; 0x1df, 0x1df; 0x1e1, 0x1e1; 0x1e3, 0x1e3; 0x1e5, 0x1e5; 0x1e7, 0x1e7; 0x1e9, 0x1e9; 0x1eb, 0x1eb; 0x1ed, 0x1ed; 0x1ef, 0x1f0; 0x1f3, 0x1f3; 0x1f5, 0x1f5; 0x1f9, 0x1f9; 0x1fb, 0x1fb; 0x1fd, 0x1fd; 0x1ff, 0x1ff; 0x201, 0x201; 0x203, 0x203; 0x205, 0x205; 0x207, 0x207; 0x209, 0x209; 0x20b, 0x20b; 0x20d, 0x20d; 0x20f, 0x20f; 0x211, 0x211; 0x213, 0x213; 0x215, 0x215; 0x217, 0x217; 0x219, 0x219; 0x21b, 0x21b; 0x21d, 0x21d; 0x21f, 0x21f; 0x221, 0x221; 0x223, 0x223; 0x225, 0x225; 0x227, 0x227; 0x229, 0x229; 0x22b, 0x22b; 0x22d, 0x22d; 0x22f, 0x22f; 0x231, 0x231; 0x233, 0x239; 0x23c, 0x23c; 0x23f, 0x240; 0x242, 0x242; 0x247, 0x247; 0x249, 0x249; 0x24b, 0x24b; 0x24d, 0x24d; 0x24f, 0x293; 0x296, 0x2b8; 0x2c0, 0x2c1; 0x2e0, 0x2e4; 0x345, 0x345; 0x371, 0x371; 0x373, 0x373; 0x377, 0x377; 0x37a, 0x37d; 0x390, 0x390; 0x3ac, 0x3ce; 0x3d0, 0x3d1; 0x3d5, 0x3d7; 0x3d9, 0x3d9; 0x3db, 0x3db; 0x3dd, 0x3dd; 0x3df, 0x3df; 0x3e1, 0x3e1; 0x3e3, 0x3e3; 0x3e5, 0x3e5; 0x3e7, 0x3e7; 0x3e9, 0x3e9; 0x3eb, 0x3eb; 0x3ed, 0x3ed; 0x3ef, 0x3f3; 0x3f5, 0x3f5; 0x3f8, 0x3f8; 0x3fb, 0x3fc; 0x430, 0x45f; 0x461, 0x461; 0x463, 0x463; 0x465, 0x465; 0x467, 0x467; 0x469, 0x469; 0x46b, 0x46b; 0x46d, 0x46d; 0x46f, 0x46f; 0x471, 0x471; 0x473, 0x473; 0x475, 0x475; 0x477, 0x477; 0x479, 0x479; 0x47b, 0x47b; 0x47d, 0x47d; 0x47f, 0x47f; 0x481, 0x481; 0x48b, 0x48b; 0x48d, 0x48d; 0x48f, 0x48f; 0x491, 0x491; 0x493, 0x493; 0x495, 0x495; 0x497, 0x497; 0x499, 0x499; 0x49b, 0x49b; 0x49d, 0x49d; 0x49f, 0x49f; 0x4a1, 0x4a1; 0x4a3, 0x4a3; 0x4a5, 0x4a5; 0x4a7, 0x4a7; 0x4a9, 0x4a9; 0x4ab, 0x4ab; 0x4ad, 0x4ad; 0x4af, 0x4af; 0x4b1, 0x4b1; 0x4b3, 0x4b3; 0x4b5, 0x4b5; 0x4b7, 0x4b7; 0x4b9, 0x4b9; 0x4bb, 0x4bb; 0x4bd, 0x4bd; 0x4bf, 0x4bf; 0x4c2, 0x4c2; 0x4c4, 0x4c4; 0x4c6, 0x4c6; 0x4c8, 0x4c8; 0x4ca, 0x4ca; 0x4cc, 0x4cc; 0x4ce, 0x4cf; 0x4d1, 0x4d1; 0x4d3, 0x4d3; 0x4d5, 0x4d5; 0x4d7, 0x4d7; 0x4d9, 0x4d9; 0x4db, 0x4db; 0x4dd, 0x4dd; 0x4df, 0x4df; 0x4e1, 0x4e1; 0x4e3, 0x4e3; 0x4e5, 0x4e5; 0x4e7, 0x4e7; 0x4e9, 0x4e9; 0x4eb, 0x4eb; 0x4ed, 0x4ed; 0x4ef, 0x4ef; 0x4f1, 0x4f1; 0x4f3, 0x4f3; 0x4f5, 0x4f5; 0x4f7, 0x4f7; 0x4f9, 0x4f9; 0x4fb, 0x4fb; 0x4fd, 0x4fd; 0x4ff, 0x4ff; 0x501, 0x501; 0x503, 0x503; 0x505, 0x505; 0x507, 0x507; 0x509, 0x509; 0x50b, 0x50b; 0x50d, 0x50d; 0x50f, 0x50f; 0x511, 0x511; 0x513, 0x513; 0x515, 0x515; 0x517, 0x517; 0x519, 0x519; 0x51b, 0x51b; 0x51d, 0x51d; 0x51f, 0x51f; 0x521, 0x521; 0x523, 0x523; 0x525, 0x525; 0x527, 0x527; 0x529, 0x529; 0x52b, 0x52b; 0x52d, 0x52d; 0x52f, 0x52f; 0x560, 0x588; 0x10d0, 0x10fa; 0x10fc, 0x10ff; 0x13f8, 0x13fd; 0x1c80, 0x1c88; 0x1c8a, 0x1c8a; 0x1d00, 0x1dbf; 0x1e01, 0x1e01; 0x1e03, 0x1e03; 0x1e05, 0x1e05; 0x1e07, 0x1e07; 0x1e09, 0x1e09; 0x1e0b, 0x1e0b; 0x1e0d, 0x1e0d; 0x1e0f, 0x1e0f; 0x1e11, 0x1e11; 0x1e13, 0x1e13; 0x1e15, 0x1e15; 0x1e17, 0x1e17; 0x1e19, 0x1e19; 0x1e1b, 0x1e1b; 0x1e1d, 0x1e1d; 0x1e1f, 0x1e1f; 0x1e21, 0x1e21; 0x1e23, 0x1e23; 0x1e25, 0x1e25; 0x1e27, 0x1e27; 0x1e29, 0x1e29; 0x1e2b, 0x1e2b; 0x1e2d, 0x1e2d; 0x1e2f, 0x1e2f; 0x1e31, 0x1e31; 0x1e33, 0x1e33; 0x1e35, 0x1e35; 0x1e37, 0x1e37; 0x1e39, 0x1e39; 0x1e3b, 0x1e3b; 0x1e3d, 0x1e3d; 0x1e3f, 0x1e3f; 0x1e41, 0x1e41; 0x1e43, 0x1e43; 0x1e45, 0x1e45; 0x1e47, 0x1e47; 0x1e49, 0x1e49; 0x1e4b, 0x1e4b; 0x1e4d, 0x1e4d; 0x1e4f, 0x1e4f; 0x1e51, 0x1e51; 0x1e53, 0x1e53; 0x1e55, 0x1e55; 0x1e57, 0x1e57; 0x1e59, 0x1e59; 0x1e5b, 0x1e5b; 0x1e5d, 0x1e5d; 0x1e5f, 0x1e5f; 0x1e61, 0x1e61; 0x1e63, 0x1e63; 0x1e65, 0x1e65; 0x1e67, 0x1e67; 0x1e69, 0x1e69; 0x1e6b, 0x1e6b; 0x1e6d, 0x1e6d; 0x1e6f, 0x1e6f; 0x1e71, 0x1e71; 0x1e73, 0x1e73; 0x1e75, 0x1e75; 0x1e77, 0x1e77; 0x1e79, 0x1e79; 0x1e7b, 0x1e7b; 0x1e7d, 0x1e7d; 0x1e7f, 0x1e7f; 0x1e81, 0x1e81; 0x1e83, 0x1e83; 0x1e85, 0x1e85; 0x1e87, 0x1e87; 0x1e89, 0x1e89; 0x1e8b, 0x1e8b; 0x1e8d, 0x1e8d; 0x1e8f, 0x1e8f; 0x1e91, 0x1e91; 0x1e93, 0x1e93; 0x1e95, 0x1e9d; 0x1e9f, 0x1e9f; 0x1ea1, 0x1ea1; 0x1ea3, 0x1ea3; 0x1ea5, 0x1ea5; 0x1ea7, 0x1ea7; 0x1ea9, 0x1ea9; 0x1eab, 0x1eab; 0x1ead, 0x1ead; 0x1eaf, 0x1eaf; 0x1eb1, 0x1eb1; 0x1eb3, 0x1eb3; 0x1eb5, 0x1eb5; 0x1eb7, 0x1eb7; 0x1eb9, 0x1eb9; 0x1ebb, 0x1ebb; 0x1ebd, 0x1ebd; 0x1ebf, 0x1ebf; 0x1ec1, 0x1ec1; 0x1ec3, 0x1ec3; 0x1ec5, 0x1ec5; 0x1ec7, 0x1ec7; 0x1ec9, 0x1ec9; 0x1ecb, 0x1ecb; 0x1ecd, 0x1ecd; 0x1ecf, 0x1ecf; 0x1ed1, 0x1ed1; 0x1ed3, 0x1ed3; 0x1ed5, 0x1ed5; 0x1ed7, 0x1ed7; 0x1ed9, 0x1ed9; 0x1edb, 0x1edb; 0x1edd, 0x1edd; 0x1edf, 0x1edf; 0x1ee1, 0x1ee1; 0x1ee3, 0x1ee3; 0x1ee5, 0x1ee5; 0x1ee7, 0x1ee7; 0x1ee9, 0x1ee9; 0x1eeb, 0x1eeb; 0x1eed, 0x1eed; 0x1eef, 0x1eef; 0x1ef1, 0x1ef1; 0x1ef3, 0x1ef3; 0x1ef5, 0x1ef5; 0x1ef7, 0x1ef7; 0x1ef9, 0x1ef9; 0x1efb, 0x1efb; 0x1efd, 0x1efd; 0x1eff, 0x1f07; 0x1f10, 0x1f15; 0x1f20, 0x1f27; 0x1f30, 0x1f37; 0x1f40, 0x1f45; 0x1f50, 0x1f57; 0x1f60, 0x1f67; 0x1f70, 0x1f7d; 0x1f80, 0x1f87; 0x1f90, 0x1f97; 0x1fa0, 0x1fa7; 0x1fb0, 0x1fb4; 0x1fb6, 0x1fb7; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fc7; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fd7; 0x1fe0, 0x1fe7; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ff7; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x210a, 0x210a; 0x210e, 0x210f; 0x2113, 0x2113; 0x212f, 0x212f; 0x2134, 0x2134; 0x2139, 0x2139; 0x213c, 0x213d; 0x2146, 0x2149; 0x214e, 0x214e; 0x2170, 0x217f; 0x2184, 0x2184; 0x24d0, 0x24e9; 0x2c30, 0x2c5f; 0x2c61, 0x2c61; 0x2c65, 0x2c66; 0x2c68, 0x2c68; 0x2c6a, 0x2c6a; 0x2c6c, 0x2c6c; 0x2c71, 0x2c71; 0x2c73, 0x2c74; 0x2c76, 0x2c7d; 0x2c81, 0x2c81; 0x2c83, 0x2c83; 0x2c85, 0x2c85; 0x2c87, 0x2c87; 0x2c89, 0x2c89; 0x2c8b, 0x2c8b; 0x2c8d, 0x2c8d; 0x2c8f, 0x2c8f; 0x2c91, 0x2c91; 0x2c93, 0x2c93; 0x2c95, 0x2c95; 0x2c97, 0x2c97; 0x2c99, 0x2c99; 0x2c9b, 0x2c9b; 0x2c9d, 0x2c9d; 0x2c9f, 0x2c9f; 0x2ca1, 0x2ca1; 0x2ca3, 0x2ca3; 0x2ca5, 0x2ca5; 0x2ca7, 0x2ca7; 0x2ca9, 0x2ca9; 0x2cab, 0x2cab; 0x2cad, 0x2cad; 0x2caf, 0x2caf; 0x2cb1, 0x2cb1; 0x2cb3, 0x2cb3; 0x2cb5, 0x2cb5; 0x2cb7, 0x2cb7; 0x2cb9, 0x2cb9; 0x2cbb, 0x2cbb; 0x2cbd, 0x2cbd; 0x2cbf, 0x2cbf; 0x2cc1, 0x2cc1; 0x2cc3, 0x2cc3; 0x2cc5, 0x2cc5; 0x2cc7, 0x2cc7; 0x2cc9, 0x2cc9; 0x2ccb, 0x2ccb; 0x2ccd, 0x2ccd; 0x2ccf, 0x2ccf; 0x2cd1, 0x2cd1; 0x2cd3, 0x2cd3; 0x2cd5, 0x2cd5; 0x2cd7, 0x2cd7; 0x2cd9, 0x2cd9; 0x2cdb, 0x2cdb; 0x2cdd, 0x2cdd; 0x2cdf, 0x2cdf; 0x2ce1, 0x2ce1; 0x2ce3, 0x2ce4; 0x2cec, 0x2cec; 0x2cee, 0x2cee; 0x2cf3, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0xa641, 0xa641; 0xa643, 0xa643; 0xa645, 0xa645; 0xa647, 0xa647; 0xa649, 0xa649; 0xa64b, 0xa64b; 0xa64d, 0xa64d; 0xa64f, 0xa64f; 0xa651, 0xa651; 0xa653, 0xa653; 0xa655, 0xa655; 0xa657, 0xa657; 0xa659, 0xa659; 0xa65b, 0xa65b; 0xa65d, 0xa65d; 0xa65f, 0xa65f; 0xa661, 0xa661; 0xa663, 0xa663; 0xa665, 0xa665; 0xa667, 0xa667; 0xa669, 0xa669; 0xa66b, 0xa66b; 0xa66d, 0xa66d; 0xa681, 0xa681; 0xa683, 0xa683; 0xa685, 0xa685; 0xa687, 0xa687; 0xa689, 0xa689; 0xa68b, 0xa68b; 0xa68d, 0xa68d; 0xa68f, 0xa68f; 0xa691, 0xa691; 0xa693, 0xa693; 0xa695, 0xa695; 0xa697, 0xa697; 0xa699, 0xa699; 0xa69b, 0xa69d; 0xa723, 0xa723; 0xa725, 0xa725; 0xa727, 0xa727; 0xa729, 0xa729; 0xa72b, 0xa72b; 0xa72d, 0xa72d; 0xa72f, 0xa731; 0xa733, 0xa733; 0xa735, 0xa735; 0xa737, 0xa737; 0xa739, 0xa739; 0xa73b, 0xa73b; 0xa73d, 0xa73d; 0xa73f, 0xa73f; 0xa741, 0xa741; 0xa743, 0xa743; 0xa745, 0xa745; 0xa747, 0xa747; 0xa749, 0xa749; 0xa74b, 0xa74b; 0xa74d, 0xa74d; 0xa74f, 0xa74f; 0xa751, 0xa751; 0xa753, 0xa753; 0xa755, 0xa755; 0xa757, 0xa757; 0xa759, 0xa759; 0xa75b, 0xa75b; 0xa75d, 0xa75d; 0xa75f, 0xa75f; 0xa761, 0xa761; 0xa763, 0xa763; 0xa765, 0xa765; 0xa767, 0xa767; 0xa769, 0xa769; 0xa76b, 0xa76b; 0xa76d, 0xa76d; 0xa76f, 0xa778; 0xa77a, 0xa77a; 0xa77c, 0xa77c; 0xa77f, 0xa77f; 0xa781, 0xa781; 0xa783, 0xa783; 0xa785, 0xa785; 0xa787, 0xa787; 0xa78c, 0xa78c; 0xa78e, 0xa78e; 0xa791, 0xa791; 0xa793, 0xa795; 0xa797, 0xa797; 0xa799, 0xa799; 0xa79b, 0xa79b; 0xa79d, 0xa79d; 0xa79f, 0xa79f; 0xa7a1, 0xa7a1; 0xa7a3, 0xa7a3; 0xa7a5, 0xa7a5; 0xa7a7, 0xa7a7; 0xa7a9, 0xa7a9; 0xa7af, 0xa7af; 0xa7b5, 0xa7b5; 0xa7b7, 0xa7b7; 0xa7b9, 0xa7b9; 0xa7bb, 0xa7bb; 0xa7bd, 0xa7bd; 0xa7bf, 0xa7bf; 0xa7c1, 0xa7c1; 0xa7c3, 0xa7c3; 0xa7c8, 0xa7c8; 0xa7ca, 0xa7ca; 0xa7cd, 0xa7cd; 0xa7cf, 0xa7cf; 0xa7d1, 0xa7d1; 0xa7d3, 0xa7d3; 0xa7d5, 0xa7d5; 0xa7d7, 0xa7d7; 0xa7d9, 0xa7d9; 0xa7db, 0xa7db; 0xa7f1, 0xa7f4; 0xa7f6, 0xa7f6; 0xa7f8, 0xa7fa; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabbf; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xff41, 0xff5a; 0x10428, 0x1044f; 0x104d8, 0x104fb; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x10780, 0x10780; 0x10783, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10cc0, 0x10cf2; 0x10d70, 0x10d85; 0x118c0, 0x118df; 0x16e60, 0x16e7f; 0x16ebb, 0x16ed3; 0x1d41a, 0x1d433; 0x1d44e, 0x1d454; 0x1d456, 0x1d467; 0x1d482, 0x1d49b; 0x1d4b6, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d4cf; 0x1d4ea, 0x1d503; 0x1d51e, 0x1d537; 0x1d552, 0x1d56b; 0x1d586, 0x1d59f; 0x1d5ba, 0x1d5d3; 0x1d5ee, 0x1d607; 0x1d622, 0x1d63b; 0x1d656, 0x1d66f; 0x1d68a, 0x1d6a5; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6e1; 0x1d6fc, 0x1d714; 0x1d716, 0x1d71b; 0x1d736, 0x1d74e; 0x1d750, 0x1d755; 0x1d770, 0x1d788; 0x1d78a, 0x1d78f; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7c9; 0x1d7cb, 0x1d7cb; 0x1df00, 0x1df09; 0x1df0b, 0x1df1e; 0x1df25, 0x1df2a; 0x1e030, 0x1e06d; 0x1e922, 0x1e943] let math = Cset.of_list [0x2b, 0x2b; 0x3c, 0x3e; 0x5e, 0x5e; 0x7c, 0x7c; 0x7e, 0x7e; 0xac, 0xac; 0xb1, 0xb1; 0xd7, 0xd7; 0xf7, 0xf7; 0x3d0, 0x3d2; 0x3d5, 0x3d5; 0x3f0, 0x3f1; 0x3f4, 0x3f6; 0x606, 0x608; 0x2016, 0x2016; 0x2032, 0x2034; 0x2040, 0x2040; 0x2044, 0x2044; 0x2052, 0x2052; 0x2061, 0x2064; 0x207a, 0x207e; 0x208a, 0x208e; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20e6; 0x20eb, 0x20ef; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2128, 0x2129; 0x212c, 0x212d; 0x212f, 0x2131; 0x2133, 0x2138; 0x213c, 0x2149; 0x214b, 0x214b; 0x2190, 0x21a7; 0x21a9, 0x21ae; 0x21b0, 0x21b1; 0x21b6, 0x21b7; 0x21bc, 0x21db; 0x21dd, 0x21dd; 0x21e4, 0x21e5; 0x21f4, 0x22ff; 0x2308, 0x230b; 0x2320, 0x2321; 0x237c, 0x237c; 0x239b, 0x23b5; 0x23b7, 0x23b7; 0x23d0, 0x23d0; 0x23dc, 0x23e2; 0x25a0, 0x25a1; 0x25ae, 0x25b7; 0x25bc, 0x25c1; 0x25c6, 0x25c7; 0x25ca, 0x25cb; 0x25cf, 0x25d3; 0x25e2, 0x25e2; 0x25e4, 0x25e4; 0x25e7, 0x25ec; 0x25f8, 0x25ff; 0x2605, 0x2606; 0x2640, 0x2640; 0x2642, 0x2642; 0x2660, 0x2663; 0x266d, 0x266f; 0x27c0, 0x27ff; 0x2900, 0x2aff; 0x2b30, 0x2b44; 0x2b47, 0x2b4c; 0xfb29, 0xfb29; 0xfe61, 0xfe66; 0xfe68, 0xfe68; 0xff0b, 0xff0b; 0xff1c, 0xff1e; 0xff3c, 0xff3c; 0xff3e, 0xff3e; 0xff5c, 0xff5c; 0xff5e, 0xff5e; 0xffe2, 0xffe2; 0xffe9, 0xffec; 0x10d8e, 0x10d8f; 0x1cef0, 0x1cef0; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1eef0, 0x1eef1; 0x1f8d0, 0x1f8d8] let other_alphabetic = Cset.of_list [0x345, 0x345; 0x363, 0x36f; 0x5b0, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c7; 0x610, 0x61a; 0x64b, 0x657; 0x659, 0x65f; 0x670, 0x670; 0x6d6, 0x6dc; 0x6e1, 0x6e4; 0x6e7, 0x6e8; 0x6ed, 0x6ed; 0x711, 0x711; 0x730, 0x73f; 0x7a6, 0x7b0; 0x816, 0x817; 0x81b, 0x823; 0x825, 0x827; 0x829, 0x82c; 0x897, 0x897; 0x8d4, 0x8df; 0x8e3, 0x8e9; 0x8f0, 0x903; 0x93a, 0x93b; 0x93e, 0x94c; 0x94e, 0x94f; 0x955, 0x957; 0x962, 0x963; 0x981, 0x983; 0x9be, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9cc; 0x9d7, 0x9d7; 0x9e2, 0x9e3; 0xa01, 0xa03; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4c; 0xa51, 0xa51; 0xa70, 0xa71; 0xa75, 0xa75; 0xa81, 0xa83; 0xabe, 0xac5; 0xac7, 0xac9; 0xacb, 0xacc; 0xae2, 0xae3; 0xafa, 0xafc; 0xb01, 0xb03; 0xb3e, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4c; 0xb56, 0xb57; 0xb62, 0xb63; 0xb82, 0xb82; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcc; 0xbd7, 0xbd7; 0xc00, 0xc04; 0xc3e, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4c; 0xc55, 0xc56; 0xc62, 0xc63; 0xc81, 0xc83; 0xcbe, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccc; 0xcd5, 0xcd6; 0xce2, 0xce3; 0xcf3, 0xcf3; 0xd00, 0xd03; 0xd3e, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4c; 0xd57, 0xd57; 0xd62, 0xd63; 0xd81, 0xd83; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xdf2, 0xdf3; 0xe31, 0xe31; 0xe34, 0xe3a; 0xe4d, 0xe4d; 0xeb1, 0xeb1; 0xeb4, 0xeb9; 0xebb, 0xebc; 0xecd, 0xecd; 0xf71, 0xf83; 0xf8d, 0xf97; 0xf99, 0xfbc; 0x102b, 0x1036; 0x1038, 0x1038; 0x103b, 0x103e; 0x1056, 0x1059; 0x105e, 0x1060; 0x1062, 0x1064; 0x1067, 0x106d; 0x1071, 0x1074; 0x1082, 0x108d; 0x108f, 0x108f; 0x109a, 0x109d; 0x1712, 0x1713; 0x1732, 0x1733; 0x1752, 0x1753; 0x1772, 0x1773; 0x17b6, 0x17c8; 0x1885, 0x1886; 0x18a9, 0x18a9; 0x1920, 0x192b; 0x1930, 0x1938; 0x1a17, 0x1a1b; 0x1a55, 0x1a5e; 0x1a61, 0x1a74; 0x1abf, 0x1ac0; 0x1acc, 0x1ace; 0x1b00, 0x1b04; 0x1b35, 0x1b43; 0x1b80, 0x1b82; 0x1ba1, 0x1ba9; 0x1bac, 0x1bad; 0x1be7, 0x1bf1; 0x1c24, 0x1c36; 0x1dd3, 0x1df4; 0x24b6, 0x24e9; 0x2de0, 0x2dff; 0xa674, 0xa67b; 0xa69e, 0xa69f; 0xa802, 0xa802; 0xa80b, 0xa80b; 0xa823, 0xa827; 0xa880, 0xa881; 0xa8b4, 0xa8c3; 0xa8c5, 0xa8c5; 0xa8ff, 0xa8ff; 0xa926, 0xa92a; 0xa947, 0xa952; 0xa980, 0xa983; 0xa9b4, 0xa9bf; 0xa9e5, 0xa9e5; 0xaa29, 0xaa36; 0xaa43, 0xaa43; 0xaa4c, 0xaa4d; 0xaa7b, 0xaa7d; 0xaab0, 0xaab0; 0xaab2, 0xaab4; 0xaab7, 0xaab8; 0xaabe, 0xaabe; 0xaaeb, 0xaaef; 0xaaf5, 0xaaf5; 0xabe3, 0xabea; 0xfb1e, 0xfb1e; 0x10376, 0x1037a; 0x10a01, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a0f; 0x10d24, 0x10d27; 0x10d69, 0x10d69; 0x10eab, 0x10eac; 0x10efa, 0x10efc; 0x11000, 0x11002; 0x11038, 0x11045; 0x11073, 0x11074; 0x11080, 0x11082; 0x110b0, 0x110b8; 0x110c2, 0x110c2; 0x11100, 0x11102; 0x11127, 0x11132; 0x11145, 0x11146; 0x11180, 0x11182; 0x111b3, 0x111bf; 0x111ce, 0x111cf; 0x1122c, 0x11234; 0x11237, 0x11237; 0x1123e, 0x1123e; 0x11241, 0x11241; 0x112df, 0x112e8; 0x11300, 0x11303; 0x1133e, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134c; 0x11357, 0x11357; 0x11362, 0x11363; 0x113b8, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113cd; 0x11435, 0x11441; 0x11443, 0x11445; 0x114b0, 0x114c1; 0x115af, 0x115b5; 0x115b8, 0x115be; 0x115dc, 0x115dd; 0x11630, 0x1163e; 0x11640, 0x11640; 0x116ab, 0x116b5; 0x1171d, 0x1172a; 0x1182c, 0x11838; 0x11930, 0x11935; 0x11937, 0x11938; 0x1193b, 0x1193c; 0x11940, 0x11940; 0x11942, 0x11942; 0x119d1, 0x119d7; 0x119da, 0x119df; 0x119e4, 0x119e4; 0x11a01, 0x11a0a; 0x11a35, 0x11a39; 0x11a3b, 0x11a3e; 0x11a51, 0x11a5b; 0x11a8a, 0x11a97; 0x11b60, 0x11b67; 0x11c2f, 0x11c36; 0x11c38, 0x11c3e; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d31, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d41; 0x11d43, 0x11d43; 0x11d47, 0x11d47; 0x11d8a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d96; 0x11ef3, 0x11ef6; 0x11f00, 0x11f01; 0x11f03, 0x11f03; 0x11f34, 0x11f3a; 0x11f3e, 0x11f40; 0x1611e, 0x1612e; 0x16f4f, 0x16f4f; 0x16f51, 0x16f87; 0x16f8f, 0x16f92; 0x16ff0, 0x16ff1; 0x1bc9e, 0x1bc9e; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e08f, 0x1e08f; 0x1e6e3, 0x1e6e3; 0x1e6e6, 0x1e6e6; 0x1e6ee, 0x1e6ef; 0x1e6f5, 0x1e6f5; 0x1e947, 0x1e947; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189] let other_lowercase = Cset.of_list [0xaa, 0xaa; 0xba, 0xba; 0x2b0, 0x2b8; 0x2c0, 0x2c1; 0x2e0, 0x2e4; 0x345, 0x345; 0x37a, 0x37a; 0x10fc, 0x10fc; 0x1d2c, 0x1d6a; 0x1d78, 0x1d78; 0x1d9b, 0x1dbf; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x2170, 0x217f; 0x24d0, 0x24e9; 0x2c7c, 0x2c7d; 0xa69c, 0xa69d; 0xa770, 0xa770; 0xa7f1, 0xa7f4; 0xa7f8, 0xa7f9; 0xab5c, 0xab5f; 0xab69, 0xab69; 0x10780, 0x10780; 0x10783, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x1e030, 0x1e06d] let other_math = Cset.of_list [0x5e, 0x5e; 0x3d0, 0x3d2; 0x3d5, 0x3d5; 0x3f0, 0x3f1; 0x3f4, 0x3f5; 0x2016, 0x2016; 0x2032, 0x2034; 0x2040, 0x2040; 0x2061, 0x2064; 0x207d, 0x207e; 0x208d, 0x208e; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20e6; 0x20eb, 0x20ef; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2128, 0x2129; 0x212c, 0x212d; 0x212f, 0x2131; 0x2133, 0x2138; 0x213c, 0x213f; 0x2145, 0x2149; 0x2195, 0x2199; 0x219c, 0x219f; 0x21a1, 0x21a2; 0x21a4, 0x21a5; 0x21a7, 0x21a7; 0x21a9, 0x21ad; 0x21b0, 0x21b1; 0x21b6, 0x21b7; 0x21bc, 0x21cd; 0x21d0, 0x21d1; 0x21d3, 0x21d3; 0x21d5, 0x21db; 0x21dd, 0x21dd; 0x21e4, 0x21e5; 0x2308, 0x230b; 0x23b4, 0x23b5; 0x23b7, 0x23b7; 0x23d0, 0x23d0; 0x23e2, 0x23e2; 0x25a0, 0x25a1; 0x25ae, 0x25b6; 0x25bc, 0x25c0; 0x25c6, 0x25c7; 0x25ca, 0x25cb; 0x25cf, 0x25d3; 0x25e2, 0x25e2; 0x25e4, 0x25e4; 0x25e7, 0x25ec; 0x2605, 0x2606; 0x2640, 0x2640; 0x2642, 0x2642; 0x2660, 0x2663; 0x266d, 0x266e; 0x27c5, 0x27c6; 0x27e6, 0x27ef; 0x2983, 0x2998; 0x29d8, 0x29db; 0x29fc, 0x29fd; 0xfe61, 0xfe61; 0xfe63, 0xfe63; 0xfe68, 0xfe68; 0xff3c, 0xff3c; 0xff3e, 0xff3e; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb] let other_uppercase = Cset.of_list [0x2160, 0x216f; 0x24b6, 0x24cf; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189] let uppercase = Cset.of_list [0x41, 0x5a; 0xc0, 0xd6; 0xd8, 0xde; 0x100, 0x100; 0x102, 0x102; 0x104, 0x104; 0x106, 0x106; 0x108, 0x108; 0x10a, 0x10a; 0x10c, 0x10c; 0x10e, 0x10e; 0x110, 0x110; 0x112, 0x112; 0x114, 0x114; 0x116, 0x116; 0x118, 0x118; 0x11a, 0x11a; 0x11c, 0x11c; 0x11e, 0x11e; 0x120, 0x120; 0x122, 0x122; 0x124, 0x124; 0x126, 0x126; 0x128, 0x128; 0x12a, 0x12a; 0x12c, 0x12c; 0x12e, 0x12e; 0x130, 0x130; 0x132, 0x132; 0x134, 0x134; 0x136, 0x136; 0x139, 0x139; 0x13b, 0x13b; 0x13d, 0x13d; 0x13f, 0x13f; 0x141, 0x141; 0x143, 0x143; 0x145, 0x145; 0x147, 0x147; 0x14a, 0x14a; 0x14c, 0x14c; 0x14e, 0x14e; 0x150, 0x150; 0x152, 0x152; 0x154, 0x154; 0x156, 0x156; 0x158, 0x158; 0x15a, 0x15a; 0x15c, 0x15c; 0x15e, 0x15e; 0x160, 0x160; 0x162, 0x162; 0x164, 0x164; 0x166, 0x166; 0x168, 0x168; 0x16a, 0x16a; 0x16c, 0x16c; 0x16e, 0x16e; 0x170, 0x170; 0x172, 0x172; 0x174, 0x174; 0x176, 0x176; 0x178, 0x179; 0x17b, 0x17b; 0x17d, 0x17d; 0x181, 0x182; 0x184, 0x184; 0x186, 0x187; 0x189, 0x18b; 0x18e, 0x191; 0x193, 0x194; 0x196, 0x198; 0x19c, 0x19d; 0x19f, 0x1a0; 0x1a2, 0x1a2; 0x1a4, 0x1a4; 0x1a6, 0x1a7; 0x1a9, 0x1a9; 0x1ac, 0x1ac; 0x1ae, 0x1af; 0x1b1, 0x1b3; 0x1b5, 0x1b5; 0x1b7, 0x1b8; 0x1bc, 0x1bc; 0x1c4, 0x1c4; 0x1c7, 0x1c7; 0x1ca, 0x1ca; 0x1cd, 0x1cd; 0x1cf, 0x1cf; 0x1d1, 0x1d1; 0x1d3, 0x1d3; 0x1d5, 0x1d5; 0x1d7, 0x1d7; 0x1d9, 0x1d9; 0x1db, 0x1db; 0x1de, 0x1de; 0x1e0, 0x1e0; 0x1e2, 0x1e2; 0x1e4, 0x1e4; 0x1e6, 0x1e6; 0x1e8, 0x1e8; 0x1ea, 0x1ea; 0x1ec, 0x1ec; 0x1ee, 0x1ee; 0x1f1, 0x1f1; 0x1f4, 0x1f4; 0x1f6, 0x1f8; 0x1fa, 0x1fa; 0x1fc, 0x1fc; 0x1fe, 0x1fe; 0x200, 0x200; 0x202, 0x202; 0x204, 0x204; 0x206, 0x206; 0x208, 0x208; 0x20a, 0x20a; 0x20c, 0x20c; 0x20e, 0x20e; 0x210, 0x210; 0x212, 0x212; 0x214, 0x214; 0x216, 0x216; 0x218, 0x218; 0x21a, 0x21a; 0x21c, 0x21c; 0x21e, 0x21e; 0x220, 0x220; 0x222, 0x222; 0x224, 0x224; 0x226, 0x226; 0x228, 0x228; 0x22a, 0x22a; 0x22c, 0x22c; 0x22e, 0x22e; 0x230, 0x230; 0x232, 0x232; 0x23a, 0x23b; 0x23d, 0x23e; 0x241, 0x241; 0x243, 0x246; 0x248, 0x248; 0x24a, 0x24a; 0x24c, 0x24c; 0x24e, 0x24e; 0x370, 0x370; 0x372, 0x372; 0x376, 0x376; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x38f; 0x391, 0x3a1; 0x3a3, 0x3ab; 0x3cf, 0x3cf; 0x3d2, 0x3d4; 0x3d8, 0x3d8; 0x3da, 0x3da; 0x3dc, 0x3dc; 0x3de, 0x3de; 0x3e0, 0x3e0; 0x3e2, 0x3e2; 0x3e4, 0x3e4; 0x3e6, 0x3e6; 0x3e8, 0x3e8; 0x3ea, 0x3ea; 0x3ec, 0x3ec; 0x3ee, 0x3ee; 0x3f4, 0x3f4; 0x3f7, 0x3f7; 0x3f9, 0x3fa; 0x3fd, 0x42f; 0x460, 0x460; 0x462, 0x462; 0x464, 0x464; 0x466, 0x466; 0x468, 0x468; 0x46a, 0x46a; 0x46c, 0x46c; 0x46e, 0x46e; 0x470, 0x470; 0x472, 0x472; 0x474, 0x474; 0x476, 0x476; 0x478, 0x478; 0x47a, 0x47a; 0x47c, 0x47c; 0x47e, 0x47e; 0x480, 0x480; 0x48a, 0x48a; 0x48c, 0x48c; 0x48e, 0x48e; 0x490, 0x490; 0x492, 0x492; 0x494, 0x494; 0x496, 0x496; 0x498, 0x498; 0x49a, 0x49a; 0x49c, 0x49c; 0x49e, 0x49e; 0x4a0, 0x4a0; 0x4a2, 0x4a2; 0x4a4, 0x4a4; 0x4a6, 0x4a6; 0x4a8, 0x4a8; 0x4aa, 0x4aa; 0x4ac, 0x4ac; 0x4ae, 0x4ae; 0x4b0, 0x4b0; 0x4b2, 0x4b2; 0x4b4, 0x4b4; 0x4b6, 0x4b6; 0x4b8, 0x4b8; 0x4ba, 0x4ba; 0x4bc, 0x4bc; 0x4be, 0x4be; 0x4c0, 0x4c1; 0x4c3, 0x4c3; 0x4c5, 0x4c5; 0x4c7, 0x4c7; 0x4c9, 0x4c9; 0x4cb, 0x4cb; 0x4cd, 0x4cd; 0x4d0, 0x4d0; 0x4d2, 0x4d2; 0x4d4, 0x4d4; 0x4d6, 0x4d6; 0x4d8, 0x4d8; 0x4da, 0x4da; 0x4dc, 0x4dc; 0x4de, 0x4de; 0x4e0, 0x4e0; 0x4e2, 0x4e2; 0x4e4, 0x4e4; 0x4e6, 0x4e6; 0x4e8, 0x4e8; 0x4ea, 0x4ea; 0x4ec, 0x4ec; 0x4ee, 0x4ee; 0x4f0, 0x4f0; 0x4f2, 0x4f2; 0x4f4, 0x4f4; 0x4f6, 0x4f6; 0x4f8, 0x4f8; 0x4fa, 0x4fa; 0x4fc, 0x4fc; 0x4fe, 0x4fe; 0x500, 0x500; 0x502, 0x502; 0x504, 0x504; 0x506, 0x506; 0x508, 0x508; 0x50a, 0x50a; 0x50c, 0x50c; 0x50e, 0x50e; 0x510, 0x510; 0x512, 0x512; 0x514, 0x514; 0x516, 0x516; 0x518, 0x518; 0x51a, 0x51a; 0x51c, 0x51c; 0x51e, 0x51e; 0x520, 0x520; 0x522, 0x522; 0x524, 0x524; 0x526, 0x526; 0x528, 0x528; 0x52a, 0x52a; 0x52c, 0x52c; 0x52e, 0x52e; 0x531, 0x556; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x13a0, 0x13f5; 0x1c89, 0x1c89; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1e00, 0x1e00; 0x1e02, 0x1e02; 0x1e04, 0x1e04; 0x1e06, 0x1e06; 0x1e08, 0x1e08; 0x1e0a, 0x1e0a; 0x1e0c, 0x1e0c; 0x1e0e, 0x1e0e; 0x1e10, 0x1e10; 0x1e12, 0x1e12; 0x1e14, 0x1e14; 0x1e16, 0x1e16; 0x1e18, 0x1e18; 0x1e1a, 0x1e1a; 0x1e1c, 0x1e1c; 0x1e1e, 0x1e1e; 0x1e20, 0x1e20; 0x1e22, 0x1e22; 0x1e24, 0x1e24; 0x1e26, 0x1e26; 0x1e28, 0x1e28; 0x1e2a, 0x1e2a; 0x1e2c, 0x1e2c; 0x1e2e, 0x1e2e; 0x1e30, 0x1e30; 0x1e32, 0x1e32; 0x1e34, 0x1e34; 0x1e36, 0x1e36; 0x1e38, 0x1e38; 0x1e3a, 0x1e3a; 0x1e3c, 0x1e3c; 0x1e3e, 0x1e3e; 0x1e40, 0x1e40; 0x1e42, 0x1e42; 0x1e44, 0x1e44; 0x1e46, 0x1e46; 0x1e48, 0x1e48; 0x1e4a, 0x1e4a; 0x1e4c, 0x1e4c; 0x1e4e, 0x1e4e; 0x1e50, 0x1e50; 0x1e52, 0x1e52; 0x1e54, 0x1e54; 0x1e56, 0x1e56; 0x1e58, 0x1e58; 0x1e5a, 0x1e5a; 0x1e5c, 0x1e5c; 0x1e5e, 0x1e5e; 0x1e60, 0x1e60; 0x1e62, 0x1e62; 0x1e64, 0x1e64; 0x1e66, 0x1e66; 0x1e68, 0x1e68; 0x1e6a, 0x1e6a; 0x1e6c, 0x1e6c; 0x1e6e, 0x1e6e; 0x1e70, 0x1e70; 0x1e72, 0x1e72; 0x1e74, 0x1e74; 0x1e76, 0x1e76; 0x1e78, 0x1e78; 0x1e7a, 0x1e7a; 0x1e7c, 0x1e7c; 0x1e7e, 0x1e7e; 0x1e80, 0x1e80; 0x1e82, 0x1e82; 0x1e84, 0x1e84; 0x1e86, 0x1e86; 0x1e88, 0x1e88; 0x1e8a, 0x1e8a; 0x1e8c, 0x1e8c; 0x1e8e, 0x1e8e; 0x1e90, 0x1e90; 0x1e92, 0x1e92; 0x1e94, 0x1e94; 0x1e9e, 0x1e9e; 0x1ea0, 0x1ea0; 0x1ea2, 0x1ea2; 0x1ea4, 0x1ea4; 0x1ea6, 0x1ea6; 0x1ea8, 0x1ea8; 0x1eaa, 0x1eaa; 0x1eac, 0x1eac; 0x1eae, 0x1eae; 0x1eb0, 0x1eb0; 0x1eb2, 0x1eb2; 0x1eb4, 0x1eb4; 0x1eb6, 0x1eb6; 0x1eb8, 0x1eb8; 0x1eba, 0x1eba; 0x1ebc, 0x1ebc; 0x1ebe, 0x1ebe; 0x1ec0, 0x1ec0; 0x1ec2, 0x1ec2; 0x1ec4, 0x1ec4; 0x1ec6, 0x1ec6; 0x1ec8, 0x1ec8; 0x1eca, 0x1eca; 0x1ecc, 0x1ecc; 0x1ece, 0x1ece; 0x1ed0, 0x1ed0; 0x1ed2, 0x1ed2; 0x1ed4, 0x1ed4; 0x1ed6, 0x1ed6; 0x1ed8, 0x1ed8; 0x1eda, 0x1eda; 0x1edc, 0x1edc; 0x1ede, 0x1ede; 0x1ee0, 0x1ee0; 0x1ee2, 0x1ee2; 0x1ee4, 0x1ee4; 0x1ee6, 0x1ee6; 0x1ee8, 0x1ee8; 0x1eea, 0x1eea; 0x1eec, 0x1eec; 0x1eee, 0x1eee; 0x1ef0, 0x1ef0; 0x1ef2, 0x1ef2; 0x1ef4, 0x1ef4; 0x1ef6, 0x1ef6; 0x1ef8, 0x1ef8; 0x1efa, 0x1efa; 0x1efc, 0x1efc; 0x1efe, 0x1efe; 0x1f08, 0x1f0f; 0x1f18, 0x1f1d; 0x1f28, 0x1f2f; 0x1f38, 0x1f3f; 0x1f48, 0x1f4d; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f5f; 0x1f68, 0x1f6f; 0x1fb8, 0x1fbb; 0x1fc8, 0x1fcb; 0x1fd8, 0x1fdb; 0x1fe8, 0x1fec; 0x1ff8, 0x1ffb; 0x2102, 0x2102; 0x2107, 0x2107; 0x210b, 0x210d; 0x2110, 0x2112; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x212d; 0x2130, 0x2133; 0x213e, 0x213f; 0x2145, 0x2145; 0x2160, 0x216f; 0x2183, 0x2183; 0x24b6, 0x24cf; 0x2c00, 0x2c2f; 0x2c60, 0x2c60; 0x2c62, 0x2c64; 0x2c67, 0x2c67; 0x2c69, 0x2c69; 0x2c6b, 0x2c6b; 0x2c6d, 0x2c70; 0x2c72, 0x2c72; 0x2c75, 0x2c75; 0x2c7e, 0x2c80; 0x2c82, 0x2c82; 0x2c84, 0x2c84; 0x2c86, 0x2c86; 0x2c88, 0x2c88; 0x2c8a, 0x2c8a; 0x2c8c, 0x2c8c; 0x2c8e, 0x2c8e; 0x2c90, 0x2c90; 0x2c92, 0x2c92; 0x2c94, 0x2c94; 0x2c96, 0x2c96; 0x2c98, 0x2c98; 0x2c9a, 0x2c9a; 0x2c9c, 0x2c9c; 0x2c9e, 0x2c9e; 0x2ca0, 0x2ca0; 0x2ca2, 0x2ca2; 0x2ca4, 0x2ca4; 0x2ca6, 0x2ca6; 0x2ca8, 0x2ca8; 0x2caa, 0x2caa; 0x2cac, 0x2cac; 0x2cae, 0x2cae; 0x2cb0, 0x2cb0; 0x2cb2, 0x2cb2; 0x2cb4, 0x2cb4; 0x2cb6, 0x2cb6; 0x2cb8, 0x2cb8; 0x2cba, 0x2cba; 0x2cbc, 0x2cbc; 0x2cbe, 0x2cbe; 0x2cc0, 0x2cc0; 0x2cc2, 0x2cc2; 0x2cc4, 0x2cc4; 0x2cc6, 0x2cc6; 0x2cc8, 0x2cc8; 0x2cca, 0x2cca; 0x2ccc, 0x2ccc; 0x2cce, 0x2cce; 0x2cd0, 0x2cd0; 0x2cd2, 0x2cd2; 0x2cd4, 0x2cd4; 0x2cd6, 0x2cd6; 0x2cd8, 0x2cd8; 0x2cda, 0x2cda; 0x2cdc, 0x2cdc; 0x2cde, 0x2cde; 0x2ce0, 0x2ce0; 0x2ce2, 0x2ce2; 0x2ceb, 0x2ceb; 0x2ced, 0x2ced; 0x2cf2, 0x2cf2; 0xa640, 0xa640; 0xa642, 0xa642; 0xa644, 0xa644; 0xa646, 0xa646; 0xa648, 0xa648; 0xa64a, 0xa64a; 0xa64c, 0xa64c; 0xa64e, 0xa64e; 0xa650, 0xa650; 0xa652, 0xa652; 0xa654, 0xa654; 0xa656, 0xa656; 0xa658, 0xa658; 0xa65a, 0xa65a; 0xa65c, 0xa65c; 0xa65e, 0xa65e; 0xa660, 0xa660; 0xa662, 0xa662; 0xa664, 0xa664; 0xa666, 0xa666; 0xa668, 0xa668; 0xa66a, 0xa66a; 0xa66c, 0xa66c; 0xa680, 0xa680; 0xa682, 0xa682; 0xa684, 0xa684; 0xa686, 0xa686; 0xa688, 0xa688; 0xa68a, 0xa68a; 0xa68c, 0xa68c; 0xa68e, 0xa68e; 0xa690, 0xa690; 0xa692, 0xa692; 0xa694, 0xa694; 0xa696, 0xa696; 0xa698, 0xa698; 0xa69a, 0xa69a; 0xa722, 0xa722; 0xa724, 0xa724; 0xa726, 0xa726; 0xa728, 0xa728; 0xa72a, 0xa72a; 0xa72c, 0xa72c; 0xa72e, 0xa72e; 0xa732, 0xa732; 0xa734, 0xa734; 0xa736, 0xa736; 0xa738, 0xa738; 0xa73a, 0xa73a; 0xa73c, 0xa73c; 0xa73e, 0xa73e; 0xa740, 0xa740; 0xa742, 0xa742; 0xa744, 0xa744; 0xa746, 0xa746; 0xa748, 0xa748; 0xa74a, 0xa74a; 0xa74c, 0xa74c; 0xa74e, 0xa74e; 0xa750, 0xa750; 0xa752, 0xa752; 0xa754, 0xa754; 0xa756, 0xa756; 0xa758, 0xa758; 0xa75a, 0xa75a; 0xa75c, 0xa75c; 0xa75e, 0xa75e; 0xa760, 0xa760; 0xa762, 0xa762; 0xa764, 0xa764; 0xa766, 0xa766; 0xa768, 0xa768; 0xa76a, 0xa76a; 0xa76c, 0xa76c; 0xa76e, 0xa76e; 0xa779, 0xa779; 0xa77b, 0xa77b; 0xa77d, 0xa77e; 0xa780, 0xa780; 0xa782, 0xa782; 0xa784, 0xa784; 0xa786, 0xa786; 0xa78b, 0xa78b; 0xa78d, 0xa78d; 0xa790, 0xa790; 0xa792, 0xa792; 0xa796, 0xa796; 0xa798, 0xa798; 0xa79a, 0xa79a; 0xa79c, 0xa79c; 0xa79e, 0xa79e; 0xa7a0, 0xa7a0; 0xa7a2, 0xa7a2; 0xa7a4, 0xa7a4; 0xa7a6, 0xa7a6; 0xa7a8, 0xa7a8; 0xa7aa, 0xa7ae; 0xa7b0, 0xa7b4; 0xa7b6, 0xa7b6; 0xa7b8, 0xa7b8; 0xa7ba, 0xa7ba; 0xa7bc, 0xa7bc; 0xa7be, 0xa7be; 0xa7c0, 0xa7c0; 0xa7c2, 0xa7c2; 0xa7c4, 0xa7c7; 0xa7c9, 0xa7c9; 0xa7cb, 0xa7cc; 0xa7ce, 0xa7ce; 0xa7d0, 0xa7d0; 0xa7d2, 0xa7d2; 0xa7d4, 0xa7d4; 0xa7d6, 0xa7d6; 0xa7d8, 0xa7d8; 0xa7da, 0xa7da; 0xa7dc, 0xa7dc; 0xa7f5, 0xa7f5; 0xff21, 0xff3a; 0x10400, 0x10427; 0x104b0, 0x104d3; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10c80, 0x10cb2; 0x10d50, 0x10d65; 0x118a0, 0x118bf; 0x16e40, 0x16e5f; 0x16ea0, 0x16eb8; 0x1d400, 0x1d419; 0x1d434, 0x1d44d; 0x1d468, 0x1d481; 0x1d49c, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b5; 0x1d4d0, 0x1d4e9; 0x1d504, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d538, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d56c, 0x1d585; 0x1d5a0, 0x1d5b9; 0x1d5d4, 0x1d5ed; 0x1d608, 0x1d621; 0x1d63c, 0x1d655; 0x1d670, 0x1d689; 0x1d6a8, 0x1d6c0; 0x1d6e2, 0x1d6fa; 0x1d71c, 0x1d734; 0x1d756, 0x1d76e; 0x1d790, 0x1d7a8; 0x1d7ca, 0x1d7ca; 0x1e900, 0x1e921; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189] let white_space = Cset.of_list [0x9, 0xd; 0x20, 0x20; 0x85, 0x85; 0xa0, 0xa0; 0x1680, 0x1680; 0x2000, 0x200a; 0x2028, 0x2029; 0x202f, 0x202f; 0x205f, 0x205f; 0x3000, 0x3000] let xid_continue = Cset.of_list [0x30, 0x39; 0x41, 0x5a; 0x5f, 0x5f; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xb7, 0xb7; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x300, 0x374; 0x376, 0x377; 0x37b, 0x37d; 0x37f, 0x37f; 0x386, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x483, 0x487; 0x48a, 0x52f; 0x531, 0x556; 0x559, 0x559; 0x560, 0x588; 0x591, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c7; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x610, 0x61a; 0x620, 0x669; 0x66e, 0x6d3; 0x6d5, 0x6dc; 0x6df, 0x6e8; 0x6ea, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x74a; 0x74d, 0x7b1; 0x7c0, 0x7f5; 0x7fa, 0x7fa; 0x7fd, 0x7fd; 0x800, 0x82d; 0x840, 0x85b; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x897, 0x8e1; 0x8e3, 0x963; 0x966, 0x96f; 0x971, 0x983; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bc, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9ce; 0x9d7, 0x9d7; 0x9dc, 0x9dd; 0x9df, 0x9e3; 0x9e6, 0x9f1; 0x9fc, 0x9fc; 0x9fe, 0x9fe; 0xa01, 0xa03; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa3c, 0xa3c; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4d; 0xa51, 0xa51; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa66, 0xa75; 0xa81, 0xa83; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabc, 0xac5; 0xac7, 0xac9; 0xacb, 0xacd; 0xad0, 0xad0; 0xae0, 0xae3; 0xae6, 0xaef; 0xaf9, 0xaff; 0xb01, 0xb03; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3c, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4d; 0xb55, 0xb57; 0xb5c, 0xb5d; 0xb5f, 0xb63; 0xb66, 0xb6f; 0xb71, 0xb71; 0xb82, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcd; 0xbd0, 0xbd0; 0xbd7, 0xbd7; 0xbe6, 0xbef; 0xc00, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3c, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4d; 0xc55, 0xc56; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc63; 0xc66, 0xc6f; 0xc80, 0xc83; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbc, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccd; 0xcd5, 0xcd6; 0xcdc, 0xcde; 0xce0, 0xce3; 0xce6, 0xcef; 0xcf1, 0xcf3; 0xd00, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4e; 0xd54, 0xd57; 0xd5f, 0xd63; 0xd66, 0xd6f; 0xd7a, 0xd7f; 0xd81, 0xd83; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xdca, 0xdca; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xde6, 0xdef; 0xdf2, 0xdf3; 0xe01, 0xe3a; 0xe40, 0xe4e; 0xe50, 0xe59; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xec8, 0xece; 0xed0, 0xed9; 0xedc, 0xedf; 0xf00, 0xf00; 0xf18, 0xf19; 0xf20, 0xf29; 0xf35, 0xf35; 0xf37, 0xf37; 0xf39, 0xf39; 0xf3e, 0xf47; 0xf49, 0xf6c; 0xf71, 0xf84; 0xf86, 0xf97; 0xf99, 0xfbc; 0xfc6, 0xfc6; 0x1000, 0x1049; 0x1050, 0x109d; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x135d, 0x135f; 0x1369, 0x1371; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1715; 0x171f, 0x1734; 0x1740, 0x1753; 0x1760, 0x176c; 0x176e, 0x1770; 0x1772, 0x1773; 0x1780, 0x17d3; 0x17d7, 0x17d7; 0x17dc, 0x17dd; 0x17e0, 0x17e9; 0x180b, 0x180d; 0x180f, 0x1819; 0x1820, 0x1878; 0x1880, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1920, 0x192b; 0x1930, 0x193b; 0x1946, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x19d0, 0x19da; 0x1a00, 0x1a1b; 0x1a20, 0x1a5e; 0x1a60, 0x1a7c; 0x1a7f, 0x1a89; 0x1a90, 0x1a99; 0x1aa7, 0x1aa7; 0x1ab0, 0x1abd; 0x1abf, 0x1add; 0x1ae0, 0x1aeb; 0x1b00, 0x1b4c; 0x1b50, 0x1b59; 0x1b6b, 0x1b73; 0x1b80, 0x1bf3; 0x1c00, 0x1c37; 0x1c40, 0x1c49; 0x1c4d, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1cd0, 0x1cd2; 0x1cd4, 0x1cfa; 0x1d00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x200c, 0x200d; 0x203f, 0x2040; 0x2054, 0x2054; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20f0; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d7f, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x2de0, 0x2dff; 0x3005, 0x3007; 0x3021, 0x302f; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x3099, 0x309a; 0x309d, 0x309f; 0x30a1, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa62b; 0xa640, 0xa66f; 0xa674, 0xa67d; 0xa67f, 0xa6f1; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dc; 0xa7f1, 0xa827; 0xa82c, 0xa82c; 0xa840, 0xa873; 0xa880, 0xa8c5; 0xa8d0, 0xa8d9; 0xa8e0, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa92d; 0xa930, 0xa953; 0xa960, 0xa97c; 0xa980, 0xa9c0; 0xa9cf, 0xa9d9; 0xa9e0, 0xa9fe; 0xaa00, 0xaa36; 0xaa40, 0xaa4d; 0xaa50, 0xaa59; 0xaa60, 0xaa76; 0xaa7a, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaef; 0xaaf2, 0xaaf6; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabea; 0xabec, 0xabed; 0xabf0, 0xabf9; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfc5d; 0xfc64, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdf9; 0xfe00, 0xfe0f; 0xfe20, 0xfe2f; 0xfe33, 0xfe34; 0xfe4d, 0xfe4f; 0xfe71, 0xfe71; 0xfe73, 0xfe73; 0xfe77, 0xfe77; 0xfe79, 0xfe79; 0xfe7b, 0xfe7b; 0xfe7d, 0xfe7d; 0xfe7f, 0xfefc; 0xff10, 0xff19; 0xff21, 0xff3a; 0xff3f, 0xff3f; 0xff41, 0xff5a; 0xff65, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x101fd, 0x101fd; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x102e0, 0x102e0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x1037a; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104a0, 0x104a9; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a38, 0x10a3a; 0x10a3f, 0x10a3f; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae6; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d27; 0x10d30, 0x10d39; 0x10d40, 0x10d65; 0x10d69, 0x10d6d; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eab, 0x10eac; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10efa, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f50; 0x10f70, 0x10f85; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11000, 0x11046; 0x11066, 0x11075; 0x1107f, 0x110ba; 0x110c2, 0x110c2; 0x110d0, 0x110e8; 0x110f0, 0x110f9; 0x11100, 0x11134; 0x11136, 0x1113f; 0x11144, 0x11147; 0x11150, 0x11173; 0x11176, 0x11176; 0x11180, 0x111c4; 0x111c9, 0x111cc; 0x111ce, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x11237; 0x1123e, 0x11241; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112ea; 0x112f0, 0x112f9; 0x11300, 0x11303; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133b, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134d; 0x11350, 0x11350; 0x11357, 0x11357; 0x1135d, 0x11363; 0x11366, 0x1136c; 0x11370, 0x11374; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113d3; 0x113e1, 0x113e2; 0x11400, 0x1144a; 0x11450, 0x11459; 0x1145e, 0x11461; 0x11480, 0x114c5; 0x114c7, 0x114c7; 0x114d0, 0x114d9; 0x11580, 0x115b5; 0x115b8, 0x115c0; 0x115d8, 0x115dd; 0x11600, 0x11640; 0x11644, 0x11644; 0x11650, 0x11659; 0x11680, 0x116b8; 0x116c0, 0x116c9; 0x116d0, 0x116e3; 0x11700, 0x1171a; 0x1171d, 0x1172b; 0x11730, 0x11739; 0x11740, 0x11746; 0x11800, 0x1183a; 0x118a0, 0x118e9; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x11935; 0x11937, 0x11938; 0x1193b, 0x11943; 0x11950, 0x11959; 0x119a0, 0x119a7; 0x119aa, 0x119d7; 0x119da, 0x119e1; 0x119e3, 0x119e4; 0x11a00, 0x11a3e; 0x11a47, 0x11a47; 0x11a50, 0x11a99; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b60, 0x11b67; 0x11bc0, 0x11be0; 0x11bf0, 0x11bf9; 0x11c00, 0x11c08; 0x11c0a, 0x11c36; 0x11c38, 0x11c40; 0x11c50, 0x11c59; 0x11c72, 0x11c8f; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d47; 0x11d50, 0x11d59; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d98; 0x11da0, 0x11da9; 0x11db0, 0x11ddb; 0x11de0, 0x11de9; 0x11ee0, 0x11ef6; 0x11f00, 0x11f10; 0x11f12, 0x11f3a; 0x11f3e, 0x11f42; 0x11f50, 0x11f5a; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246e; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13440, 0x13455; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x16139; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a60, 0x16a69; 0x16a70, 0x16abe; 0x16ac0, 0x16ac9; 0x16ad0, 0x16aed; 0x16af0, 0x16af4; 0x16b00, 0x16b36; 0x16b40, 0x16b43; 0x16b50, 0x16b59; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16d70, 0x16d79; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f4f, 0x16f87; 0x16f8f, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe4; 0x16ff0, 0x16ff6; 0x17000, 0x18cd5; 0x18cff, 0x18d1e; 0x18d80, 0x18df2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b122; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b167; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1bc9d, 0x1bc9e; 0x1ccf0, 0x1ccf9; 0x1cf00, 0x1cf2d; 0x1cf30, 0x1cf46; 0x1d165, 0x1d169; 0x1d16d, 0x1d172; 0x1d17b, 0x1d182; 0x1d185, 0x1d18b; 0x1d1aa, 0x1d1ad; 0x1d242, 0x1d244; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1da00, 0x1da36; 0x1da3b, 0x1da6c; 0x1da75, 0x1da75; 0x1da84, 0x1da84; 0x1da9b, 0x1da9f; 0x1daa1, 0x1daaf; 0x1df00, 0x1df1e; 0x1df25, 0x1df2a; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e030, 0x1e06d; 0x1e08f, 0x1e08f; 0x1e100, 0x1e12c; 0x1e130, 0x1e13d; 0x1e140, 0x1e149; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ae; 0x1e2c0, 0x1e2f9; 0x1e4d0, 0x1e4f9; 0x1e5d0, 0x1e5fa; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6f5; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e8d0, 0x1e8d6; 0x1e900, 0x1e94b; 0x1e950, 0x1e959; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1fbf0, 0x1fbf9; 0x20000, 0x2a6df; 0x2a700, 0x2b81d; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0xe0100, 0xe01ef] let xid_start = Cset.of_list [0x41, 0x5a; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x370, 0x374; 0x376, 0x377; 0x37b, 0x37d; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x48a, 0x52f; 0x531, 0x556; 0x559, 0x559; 0x560, 0x588; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x620, 0x64a; 0x66e, 0x66f; 0x671, 0x6d3; 0x6d5, 0x6d5; 0x6e5, 0x6e6; 0x6ee, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x710; 0x712, 0x72f; 0x74d, 0x7a5; 0x7b1, 0x7b1; 0x7ca, 0x7ea; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x800, 0x815; 0x81a, 0x81a; 0x824, 0x824; 0x828, 0x828; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x8a0, 0x8c9; 0x904, 0x939; 0x93d, 0x93d; 0x950, 0x950; 0x958, 0x961; 0x971, 0x980; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9bd; 0x9ce, 0x9ce; 0x9dc, 0x9dd; 0x9df, 0x9e1; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa72, 0xa74; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xabd; 0xad0, 0xad0; 0xae0, 0xae1; 0xaf9, 0xaf9; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb3d; 0xb5c, 0xb5d; 0xb5f, 0xb61; 0xb71, 0xb71; 0xb83, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbd0, 0xbd0; 0xc05, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc3d; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc61; 0xc80, 0xc80; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcbd; 0xcdc, 0xcde; 0xce0, 0xce1; 0xcf1, 0xcf2; 0xd04, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd3d; 0xd4e, 0xd4e; 0xd54, 0xd56; 0xd5f, 0xd61; 0xd7a, 0xd7f; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xe01, 0xe30; 0xe32, 0xe32; 0xe40, 0xe46; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb0; 0xeb2, 0xeb2; 0xebd, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf88, 0xf8c; 0x1000, 0x102a; 0x103f, 0x103f; 0x1050, 0x1055; 0x105a, 0x105d; 0x1061, 0x1061; 0x1065, 0x1066; 0x106e, 0x1070; 0x1075, 0x1081; 0x108e, 0x108e; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1711; 0x171f, 0x1731; 0x1740, 0x1751; 0x1760, 0x176c; 0x176e, 0x1770; 0x1780, 0x17b3; 0x17d7, 0x17d7; 0x17dc, 0x17dc; 0x1820, 0x1878; 0x1880, 0x18a8; 0x18aa, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a16; 0x1a20, 0x1a54; 0x1aa7, 0x1aa7; 0x1b05, 0x1b33; 0x1b45, 0x1b4c; 0x1b83, 0x1ba0; 0x1bae, 0x1baf; 0x1bba, 0x1be5; 0x1c00, 0x1c23; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x1d00, 0x1dbf; 0x1e00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209c; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cee; 0x2cf2, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x3005, 0x3007; 0x3021, 0x3029; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x309d, 0x309f; 0x30a1, 0x30fa; 0x30fc, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa640, 0xa66e; 0xa67f, 0xa69d; 0xa6a0, 0xa6ef; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dc; 0xa7f1, 0xa801; 0xa803, 0xa805; 0xa807, 0xa80a; 0xa80c, 0xa822; 0xa840, 0xa873; 0xa882, 0xa8b3; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8fe; 0xa90a, 0xa925; 0xa930, 0xa946; 0xa960, 0xa97c; 0xa984, 0xa9b2; 0xa9cf, 0xa9cf; 0xa9e0, 0xa9e4; 0xa9e6, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa28; 0xaa40, 0xaa42; 0xaa44, 0xaa4b; 0xaa60, 0xaa76; 0xaa7a, 0xaa7a; 0xaa7e, 0xaaaf; 0xaab1, 0xaab1; 0xaab5, 0xaab6; 0xaab9, 0xaabd; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaea; 0xaaf2, 0xaaf4; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabe2; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb1d; 0xfb1f, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfc5d; 0xfc64, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdf9; 0xfe71, 0xfe71; 0xfe73, 0xfe73; 0xfe77, 0xfe77; 0xfe79, 0xfe79; 0xfe7b, 0xfe7b; 0xfe7d, 0xfe7d; 0xfe7f, 0xfefc; 0xff21, 0xff3a; 0xff41, 0xff5a; 0xff66, 0xff9d; 0xffa0, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x10375; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107ba; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a00; 0x10a10, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d23; 0x10d4a, 0x10d65; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11003, 0x11037; 0x11071, 0x11072; 0x11075, 0x11075; 0x11083, 0x110af; 0x110d0, 0x110e8; 0x11103, 0x11126; 0x11144, 0x11144; 0x11147, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11183, 0x111b2; 0x111c1, 0x111c4; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x1122b; 0x1123f, 0x11240; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112de; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x1133d; 0x11350, 0x11350; 0x1135d, 0x11361; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113b7; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11434; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114af; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115ae; 0x115d8, 0x115db; 0x11600, 0x1162f; 0x11644, 0x11644; 0x11680, 0x116aa; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x11740, 0x11746; 0x11800, 0x1182b; 0x118a0, 0x118df; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x1192f; 0x1193f, 0x1193f; 0x11941, 0x11941; 0x119a0, 0x119a7; 0x119aa, 0x119d0; 0x119e1, 0x119e1; 0x119e3, 0x119e3; 0x11a00, 0x11a00; 0x11a0b, 0x11a32; 0x11a3a, 0x11a3a; 0x11a50, 0x11a50; 0x11a5c, 0x11a89; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c2e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d30; 0x11d46, 0x11d46; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d89; 0x11d98, 0x11d98; 0x11db0, 0x11ddb; 0x11ee0, 0x11ef2; 0x11f02, 0x11f02; 0x11f04, 0x11f10; 0x11f12, 0x11f33; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246e; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1611d; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b40, 0x16b43; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f50, 0x16f50; 0x16f93, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff2, 0x16ff6; 0x17000, 0x18cd5; 0x18cff, 0x18d1e; 0x18d80, 0x18df2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b122; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b167; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a5; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1df00, 0x1df1e; 0x1df25, 0x1df2a; 0x1e030, 0x1e06d; 0x1e100, 0x1e12c; 0x1e137, 0x1e13d; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4eb; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6e2; 0x1e6e4, 0x1e6e5; 0x1e6e7, 0x1e6ed; 0x1e6f0, 0x1e6f4; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e900, 0x1e943; 0x1e94b, 0x1e94b; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x20000, 0x2a6df; 0x2a700, 0x2b81d; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479] let list = [ ("alphabetic", alphabetic); ("ascii_hex_digit", ascii_hex_digit); ("hex_digit", hex_digit); ("id_continue", id_continue); ("id_start", id_start); ("lowercase", lowercase); ("math", math); ("other_alphabetic", other_alphabetic); ("other_lowercase", other_lowercase); ("other_math", other_math); ("other_uppercase", other_uppercase); ("uppercase", uppercase); ("white_space", white_space); ("xid_continue", xid_continue); ("xid_start", xid_start) ] (* ignoring: - bidi_control - case_ignorable - cased - changes_when_casefolded - changes_when_casemapped - changes_when_lowercased - changes_when_titlecased - changes_when_uppercased - dash - default_ignorable_code_point - deprecated - diacritic - extender - grapheme_base - grapheme_extend - grapheme_link - hyphen - id_compat_math_continue - id_compat_math_start - ideographic - ids_binary_operator - ids_trinary_operator - ids_unary_operator - join_control - logical_order_exception - modifier_combining_mark - noncharacter_code_point - other_default_ignorable_code_point - other_grapheme_extend - other_id_continue - other_id_start - pattern_syntax - pattern_white_space - prepended_concatenation_mark - quotation_mark - radical - regional_indicator - sentence_terminal - soft_dotted - terminal_punctuation - unified_ideograph - variation_selector *) end sedlex-3.8.1/examples/with_menhir/000077500000000000000000000000001526113672700171545ustar00rootroot00000000000000sedlex-3.8.1/examples/with_menhir/dune000066400000000000000000000003161526113672700200320ustar00rootroot00000000000000(executable (name main) (libraries sedlex menhirLib) (preprocess (pps sedlex.ppx))) (rule (alias runtest) (deps (:< main.exe)) (action (run %{<}))) (menhir (modules parser) (flags --table)) sedlex-3.8.1/examples/with_menhir/lexer.ml000066400000000000000000000025551526113672700206340ustar00rootroot00000000000000(* Sedlex lexer that produces tokens for a menhir parser. Two approaches are shown: 1. [tokenize] — for menhir's classic API (same technique as ocamlyacc): wraps the sedlex lexer into [Lexing.lexbuf -> token]. 2. [tokenize_incremental] — for menhir's incremental API: uses [Sedlexing.with_tokenizer] which returns a [unit -> token * position * position] supplier. *) let digit = [%sedlex.regexp? '0' .. '9'] let rec token buf = match%sedlex buf with | Plus digit -> Parser.INT (int_of_string (Sedlexing.Utf8.lexeme buf)) | '+' -> Parser.PLUS | '-' -> Parser.MINUS | '*' -> Parser.TIMES | '/' -> Parser.DIV | '(' -> Parser.LPAREN | ')' -> Parser.RPAREN | Plus white_space -> token buf | eof -> Parser.EOF | _ -> failwith ("Unexpected character: " ^ Sedlexing.Utf8.lexeme buf) (* Approach 1: classic API — wrap into [Lexing.lexbuf -> token] *) let tokenize buf = let lexing_lexbuf = Lexing.from_string "" in let tokenize _lexing_lexbuf = let tok = token buf in let start_pos, end_pos = Sedlexing.lexing_positions buf in _lexing_lexbuf.Lexing.lex_start_p <- start_pos; _lexing_lexbuf.Lexing.lex_curr_p <- end_pos; tok in (tokenize, lexing_lexbuf) (* Approach 2: incremental API — use [Sedlexing.with_tokenizer] *) let tokenize_incremental buf = Sedlexing.with_tokenizer token buf sedlex-3.8.1/examples/with_menhir/main.ml000066400000000000000000000010471526113672700204340ustar00rootroot00000000000000let () = let input = "1 + 2 * (3 - 1)" in (* Classic API *) let buf = Sedlexing.Utf8.from_string input in let tokenize, lexbuf = Lexer.tokenize buf in let result = Parser.main tokenize lexbuf in Printf.printf "classic: %s = %d\n" input result; (* Incremental API *) let buf = Sedlexing.Utf8.from_string input in let supplier = Lexer.tokenize_incremental buf in let result = Parser.MenhirInterpreter.loop supplier (Parser.Incremental.main Lexing.dummy_pos) in Printf.printf "incremental: %s = %d\n" input result sedlex-3.8.1/examples/with_menhir/parser.mly000066400000000000000000000007211526113672700211730ustar00rootroot00000000000000/* Simple calculator parser for use with sedlex via menhir */ %token INT %token PLUS MINUS TIMES DIV %token LPAREN RPAREN %token EOF %left PLUS MINUS %left TIMES DIV %start main %% main: | e = expr; EOF { e } ; expr: | n = INT { n } | LPAREN; e = expr; RPAREN { e } | e1 = expr; PLUS; e2 = expr { e1 + e2 } | e1 = expr; MINUS; e2 = expr { e1 - e2 } | e1 = expr; TIMES; e2 = expr { e1 * e2 } | e1 = expr; DIV; e2 = expr { e1 / e2 } ; sedlex-3.8.1/examples/with_ocamlyacc/000077500000000000000000000000001526113672700176255ustar00rootroot00000000000000sedlex-3.8.1/examples/with_ocamlyacc/dune000066400000000000000000000002531526113672700205030ustar00rootroot00000000000000(executable (name main) (libraries sedlex) (preprocess (pps sedlex.ppx))) (rule (alias runtest) (deps (:< main.exe)) (action (run %{<}))) (ocamlyacc parser) sedlex-3.8.1/examples/with_ocamlyacc/lexer.ml000066400000000000000000000024211526113672700212750ustar00rootroot00000000000000(* Sedlex lexer that produces tokens for an ocamlyacc parser. The key integration point: ocamlyacc parsers expect a function of type [Lexing.lexbuf -> token], but sedlex uses [Sedlexing.lexbuf]. We create a dummy [Lexing.lexbuf] and update its position fields after each token so that error reporting works correctly. *) let digit = [%sedlex.regexp? '0' .. '9'] let rec token buf = match%sedlex buf with | Plus digit -> Parser.INT (int_of_string (Sedlexing.Utf8.lexeme buf)) | '+' -> Parser.PLUS | '-' -> Parser.MINUS | '*' -> Parser.TIMES | '/' -> Parser.DIV | '(' -> Parser.LPAREN | ')' -> Parser.RPAREN | Plus white_space -> token buf | eof -> Parser.EOF | _ -> failwith ("Unexpected character: " ^ Sedlexing.Utf8.lexeme buf) (* Wrap a sedlex lexer for use with ocamlyacc. Returns [(tokenize, lexing_lexbuf)] where [tokenize] has type [Lexing.lexbuf -> token] as expected by the generated parser. *) let tokenize buf = let lexing_lexbuf = Lexing.from_string "" in let tokenize _lexing_lexbuf = let tok = token buf in let start_pos, end_pos = Sedlexing.lexing_positions buf in _lexing_lexbuf.Lexing.lex_start_p <- start_pos; _lexing_lexbuf.Lexing.lex_curr_p <- end_pos; tok in (tokenize, lexing_lexbuf) sedlex-3.8.1/examples/with_ocamlyacc/main.ml000066400000000000000000000003421526113672700211020ustar00rootroot00000000000000let () = let input = "1 + 2 * (3 - 1)" in let buf = Sedlexing.Utf8.from_string input in let tokenize, lexbuf = Lexer.tokenize buf in let result = Parser.main tokenize lexbuf in Printf.printf "%s = %d\n" input result sedlex-3.8.1/examples/with_ocamlyacc/parser.mly000066400000000000000000000006431526113672700216470ustar00rootroot00000000000000/* Simple calculator parser for use with sedlex via ocamlyacc */ %token INT %token PLUS MINUS TIMES DIV %token LPAREN RPAREN %token EOF %left PLUS MINUS %left TIMES DIV %start main %type main %% main: | expr EOF { $1 } ; expr: | INT { $1 } | LPAREN expr RPAREN { $2 } | expr PLUS expr { $1 + $3 } | expr MINUS expr { $1 - $3 } | expr TIMES expr { $1 * $3 } | expr DIV expr { $1 / $3 } ; sedlex-3.8.1/sedlex.opam000066400000000000000000000022611526113672700151640ustar00rootroot00000000000000# This file is generated by dune, edit dune-project instead opam-version: "2.0" version: "3.8.1" synopsis: "An OCaml lexer generator for Unicode" description: """ sedlex is a lexer generator for OCaml. It is similar to ocamllex, but supports Unicode. Unlike ocamllex, sedlex allows lexer specifications within regular OCaml source files. Lexing specific constructs are provided via a ppx syntax extension.""" maintainer: ["Alain Frisch "] authors: [ "Alain Frisch " "https://github.com/ocaml-community/sedlex/graphs/contributors" ] license: "MIT" homepage: "https://github.com/ocaml-community/sedlex" doc: "https://ocaml-community.github.io/sedlex/index.html" bug-reports: "https://github.com/ocaml-community/sedlex/issues" depends: [ "ocaml" {>= "4.11"} "dune" {>= "3.18"} "ppxlib" {>= "0.26.0"} "gen" "ppx_expect" {with-test} "menhir" {with-test} "odoc" {with-doc} ] build: [ ["dune" "subst"] {dev} [ "dune" "build" "-p" name "-j" jobs "@install" "@runtest" {with-test} "@doc" {with-doc} ] ] dev-repo: "git+https://github.com/ocaml-community/sedlex.git" x-maintenance-intent: ["(latest)"] sedlex-3.8.1/src/000077500000000000000000000000001526113672700136105ustar00rootroot00000000000000sedlex-3.8.1/src/compiler/000077500000000000000000000000001526113672700154225ustar00rootroot00000000000000sedlex-3.8.1/src/compiler/cset.ml000066400000000000000000000050631526113672700167160ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Character sets are represented as lists of intervals. The intervals must be non-overlapping and not collapsable, and the list must be ordered in increasing order. *) type t = (int * int) list let rec range_to_seq a b next () = if a = b then Seq.Cons (a, next) else Seq.Cons (a, range_to_seq (a + 1) b next) let rec to_seq x () = match x with [] -> Seq.Nil | (a, b) :: xs -> range_to_seq a b (to_seq xs) () let check_invariant l = let rec loop prev = function | [] -> () | (a, b) :: xs -> if a < prev then failwith (Printf.sprintf "Cset.of_list: not in increasing order or overlapping. \ [_-%d]-[%d-%d]" prev a b); if a = prev then failwith (Printf.sprintf "Cset.of_list: adjacent range. [_-%d]-[%d-%d]" prev a b); if a > b then failwith (Printf.sprintf "Cset.of_list: malformed range. [%d-%d]" a b); loop b xs in loop (-1) l let of_list l = check_invariant l; l let to_list l = l let max_code = 0x10ffff (* must be < max_int *) let min_code = -1 let empty = [] let singleton i = [(i, i)] let is_empty = function [] -> true | _ -> false let interval i j = if i <= j then [(i, j)] else [(j, i)] let eof = singleton (-1) let any = interval 0 max_code let rec union c1 c2 = match (c1, c2) with | [], _ -> c2 | _, [] -> c1 | ((i1, j1) as s1) :: r1, (i2, j2) :: r2 -> if i1 <= i2 then if j1 + 1 < i2 then s1 :: union r1 c2 else if j1 < j2 then union r1 ((i1, j2) :: r2) else union c1 r2 else union c2 c1 let union_list : t list -> t = function | [] -> empty | [x] -> x | l -> List.concat l |> List.sort (fun a b -> compare b a) |> List.fold_left (fun (acc : t) (x : int * int) -> union [x] acc) empty let complement c = let rec aux start = function | [] -> if start <= max_code then [(start, max_code)] else [] | (i, j) :: l -> (start, i - 1) :: aux (succ j) l in match c with (-1, j) :: l -> aux (succ j) l | l -> aux (-1) l let rec mem c = function | [] -> false | (lo, hi) :: rest -> if c < lo then false else if c <= hi then true else mem c rest let intersection c1 c2 = complement (union (complement c1) (complement c2)) let difference c1 c2 = complement (union (complement c1) c2) sedlex-3.8.1/src/compiler/cset.mli000066400000000000000000000016011526113672700170610ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (** Representation of sets of unicode code points. *) (** Character sets are represented as lists of intervals. The intervals must be non-overlapping and not collapsable, and the list must be ordered in increasing order. *) type t = private (int * int) list val of_list : (int * int) list -> t val to_list : t -> (int * int) list val min_code : int val max_code : int val empty : t val any : t val union : t -> t -> t val union_list : t list -> t val difference : t -> t -> t val intersection : t -> t -> t val is_empty : t -> bool val eof : t val singleton : int -> t val interval : int -> int -> t val mem : int -> t -> bool val to_seq : t -> int Seq.t sedlex-3.8.1/src/compiler/dune000066400000000000000000000001011526113672700162700ustar00rootroot00000000000000(library (name sedlex_compiler) (public_name sedlex.compiler)) sedlex-3.8.1/src/compiler/ir.ml000066400000000000000000000141541526113672700163730ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2026, Hugo Heuzard *) type t = | Chars of Cset.t | Seq of t list | Alt of t list | Star of t | Plus of t | Rep of t * int * int | Eps | Capture of string * t module SSet = Set.Make (String) let rec capture_names_acc acc = function | Chars _ | Eps -> acc | Capture (name, inner) -> capture_names_acc (SSet.add name acc) inner | Seq elems -> List.fold_left capture_names_acc acc elems | Alt branches -> List.fold_left capture_names_acc acc branches | Star inner | Plus inner | Rep (inner, _, _) -> capture_names_acc acc inner let capture_names t = capture_names_acc SSet.empty t let reject_captures ctx t = if SSet.is_empty (capture_names t) then Ok t else Error (Printf.sprintf "'as' bindings are not supported inside %s" ctx) (* Analysis *) let rec fixed_length = function | Chars c -> (* [eof] (code point -1) is zero-width at runtime: [next] reports EOF without advancing [pos]. A pure-eof cset is width 0; a cset mixing eof with real characters has a data-dependent width (0 or 1), so its length is unknown. *) if Cset.mem (-1) c then if Cset.is_empty (Cset.difference c Cset.eof) then Some 0 else None else Some 1 | Eps -> Some 0 | Capture (_, inner) -> fixed_length inner | Seq elems -> List.fold_left (fun acc e -> match (acc, fixed_length e) with | Some a, Some b -> Some (a + b) | _ -> None) (Some 0) elems | Alt branches -> ( match List.map fixed_length branches with | [] -> None | first :: rest -> if List.for_all (Option.equal Int.equal first) rest then first else None) | Rep (inner, n, m) -> if n = m then ( match fixed_length inner with Some l -> Some (n * l) | None -> None) else None | Star _ | Plus _ -> None (* Smart constructors *) let chars c = Chars c let eps = Eps let capture name inner = if SSet.mem name (capture_names inner) then Error (Printf.sprintf "'as' binding '%s' shadows an inner binding of the same name" name) else Ok (Capture (name, inner)) let star t = match reject_captures "Star" t with Error _ as e -> e | Ok t -> Ok (Star t) let plus t = match reject_captures "Plus" t with Error _ as e -> e | Ok t -> Ok (Plus t) let rep t n m = if not (0 <= n && n <= m) then Error "Invalid range for Rep operator" else ( match reject_captures "Rep" t with | Error _ as e -> e (* [Rep (t, 1, 1)] is just [t]; normalize so single-character-length checks (Compl/Sub/Intersect) still see the bare [Chars] node, as they did on master where [repeat r (1, 1)] desugared to [r]. *) | Ok t -> if n = 1 && m = 1 then Ok t else Ok (Rep (t, n, m))) let seq a b = let an = capture_names a in let bn = capture_names b in let dups = SSet.inter an bn in if not (SSet.is_empty dups) then Error (Printf.sprintf "'as' binding '%s' is bound in multiple positions" (SSet.choose dups)) else Ok (match (a, b) with | Eps, x | x, Eps -> x | Seq l1, Seq l2 -> Seq (l1 @ l2) | Seq l1, x -> Seq (l1 @ [x]) | x, Seq l2 -> Seq (x :: l2) | _ -> Seq [a; b]) let alt a b = let an = capture_names a in let bn = capture_names b in if not (SSet.equal an bn) then Error "all branches of '|' must bind the same names with 'as'" else Ok (match (a, b) with | Chars c1, Chars c2 -> Chars (Cset.union c1 c2) | Alt l1, Alt l2 -> Alt (l1 @ l2) | Alt l1, x -> Alt (l1 @ [x]) | x, Alt l2 -> Alt (x :: l2) | _ -> Alt [a; b]) (* All structural constraints are enforced by the smart constructors. [check_invariant] verifies them as a debug assertion. *) let check_invariant t = let rec check = function | Chars _ | Eps -> SSet.empty | Capture (name, inner) -> let names = check inner in assert (not (SSet.mem name names)); SSet.add name names | Seq elems -> assert (List.length elems >= 2); List.fold_left (fun acc x -> let names = check x in assert (SSet.is_empty (SSet.inter acc names)); SSet.union acc names) SSet.empty elems | Alt [] | Alt [_] -> assert false | Alt (first :: rest) -> let names = check first in List.iter (fun x -> assert (SSet.equal names (check x))) rest; names | Star inner | Plus inner -> assert (SSet.is_empty (check inner)); SSet.empty | Rep (inner, _, _) -> assert (SSet.is_empty (check inner)); SSet.empty in let _ : SSet.t = check t in () (* Pretty-printing *) let rec pp fmt = function | Chars cset -> ( let intervals = (cset : Cset.t :> (int * int) list) in let cp fmt c = if c >= 32 && c <= 126 then Format.fprintf fmt "'%c'" (Char.chr c) else Format.fprintf fmt "0x%04X" c in match intervals with | [(c, c')] when c = c' -> cp fmt c | _ -> Format.fprintf fmt "[%a]" (Format.pp_print_list ~pp_sep:(fun fmt () -> Format.fprintf fmt ", ") (fun fmt (lo, hi) -> if lo = hi then cp fmt lo else Format.fprintf fmt "%a-%a" cp lo cp hi)) intervals) | Eps -> Format.fprintf fmt "eps" | Seq elems -> Format.fprintf fmt "(%a)" (Format.pp_print_list ~pp_sep:(fun fmt () -> Format.fprintf fmt ", ") pp) elems | Alt branches -> Format.fprintf fmt "(%a)" (Format.pp_print_list ~pp_sep:(fun fmt () -> Format.fprintf fmt " | ") pp) branches | Star inner -> Format.fprintf fmt "Star %a" pp inner | Plus inner -> Format.fprintf fmt "Plus %a" pp inner | Rep (inner, n, m) -> Format.fprintf fmt "Rep(%a, %d..%d)" pp inner n m | Capture (name, inner) -> Format.fprintf fmt "(%a as %s)" pp inner name let show t = Format.asprintf "%a" pp t sedlex-3.8.1/src/compiler/ir.mli000066400000000000000000000054061526113672700165440ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2026, Hugo Heuzard *) (** Intermediate representation for sedlex patterns. This IR captures the regexp structure including named capture annotations ([as] bindings) but {b before} tag allocation, fixed-length optimization, or discriminator handling. The compiler processes this IR to produce a compiled DFA with binding extraction information. The IR does not depend on ppxlib and can be constructed directly in tests. *) (** A pattern with named captures. *) type t = | Chars of Cset.t (** Match a single code point in the given set. *) | Seq of t list (** Sequence (concatenation). Invariant: length >= 2. Use {!seq} smart constructor. *) | Alt of t list (** Alternation. Invariant: length >= 2. Use {!alt} smart constructor which flattens nested [Alt]s. *) | Star of t (** Kleene star (zero or more). *) | Plus of t (** One or more. *) | Rep of t * int * int (** [Rep (r, n, m)]: between [n] and [m] repetitions. *) | Eps (** Empty string (epsilon). *) | Capture of string * t (** Named capture: [Capture (name, inner)] wraps [inner] with an [as] binding. The compiler decides tag allocation strategy. *) (** {2 Smart constructors} Constructors that enforce structural invariants return [result]. - {!alt} checks name consistency across branches. - {!capture} checks for shadowed inner bindings. - {!seq} rejects duplicate capture names across elements. - {!star}, {!plus}, {!rep} reject inner captures. *) val chars : Cset.t -> t val seq : t -> t -> (t, string) result val alt : t -> t -> (t, string) result val star : t -> (t, string) result val plus : t -> (t, string) result val rep : t -> int -> int -> (t, string) result val eps : t val capture : string -> t -> (t, string) result (** [reject_captures ctx t] returns [Ok t] if [t] contains no [Capture] nodes, or [Error msg] mentioning [ctx] otherwise. *) val reject_captures : string -> t -> (t, string) result (** {2 Analysis} *) module SSet : Set.S with type elt = string (** [capture_names t] returns the set of capture names in [t]. *) val capture_names : t -> SSet.t (** [fixed_length t] returns [Some n] if [t] always matches exactly [n] code points, or [None] if the length is variable. *) val fixed_length : t -> int option (** {2 Invariant checking} All structural constraints are enforced by the smart constructors. [check_invariant] asserts these hold. Use for debugging. *) val check_invariant : t -> unit (** {2 Pretty-printing} *) val pp : Format.formatter -> t -> unit val show : t -> string sedlex-3.8.1/src/compiler/sedlex.ml000066400000000000000000000734471526113672700172570ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (* Implementation overview ======================= Sedlex compiles regular expressions to Tagged DFAs. 1. NFA construction (type regexp = node -> node) Each regexp combinator (chars, seq, alt, rep, ...) is a function that, given a successor node, builds a fragment of NFA and returns its entry node. This continuation-passing style makes sequencing natural (seq is just function composition) and avoids explicit epsilon nodes for concatenation. 2. Tags for `as` bindings (Laurikari-style) NFA nodes may carry a tag operation (Set_position or Set_value). [bind] wraps a sub-regexp with start/end tagged epsilon nodes so the DFA can record sub-match positions at runtime. When the PPX can compute one boundary from a known offset (see [pos_expr] in ppx_sedlex.ml), [bind_start_only] or [bind_end_only] is used instead, saving a memory cell. Discriminator tags (Set_value) disambiguate or-patterns where multiple branches bind the same name. 3. Determinization (compile) Subset construction for tagged NFAs (Laurikari, "NFAs with Tagged Transitions", 2000), with the one-symbol lookahead of Trofimovich, "Tagged Deterministic Finite Automata with Lookahead" (2017): a TDFA(1). - A DFA state is an ordered list of configurations (NFA node, tag -> register map). One map per path, rather than one shared vector, keeps captures correct when a tagged node is reachable from several paths at once, e.g. a Star loop whose closure contains the start of the following capture. - Order is priority. The epsilon closure is a DFS along [eps] lists and the first path to reach a node wins: leftmost-greedy disambiguation among the parses of the longest match. - Lookahead: the tag writes met in a state's closure are delayed. A configuration performs them when it takes a character transition; one that cannot consume the next character never does. So the end of [Plus digit as n] is written once, on leaving the loop. - Transition operations execute before their character is consumed, final operations on entering the state. - End of input is a transition of its own, read once: it leads to a state without transitions. It does not advance the position but counts as one more symbol read, so a match that reads it is longer than the state's own accept. - Each tag write of a transition gets a fresh register. States are looked up modulo register renaming, with equal delayed writes; reaching an existing state emits the moves realigning the registers. The operations of a transition form a parallel move, which the code generator implements by let-binding the sources it overwrites. - Canonical cells (cell = tag id) are what the generated bindings read. Only the final operations of accepting states write them, from the accepting path's registers and delayed writes, just before [Sedlexing.mark]; working registers live above them. So the cells of the marked rule survive a failed longer match, and [Sedlexing.mark] and [Sedlexing.backtrack] save and restore nothing. Future work (see #175): register optimization as in Trofimovich 2017 (section 7), DFA minimization. *) module Cset = Cset (* NFA *) type tag_op = | Set_position of { dst : int } | Set_value of { dst : int; value : int } | Copy of { dst : int; src : int } type node = { id : int; (** Unique identifier; nodes are compared by it. *) mutable eps : node list; (** Epsilon successors (no input consumed). *) mutable trans : (Cset.t * node) list; (** Char-set-labelled transitions. *) tag : tag_op option; (** Tag operation executed when entering this node. *) } (* Compilation regexp -> NFA *) type regexp = node -> node let cur_id = ref 0 let new_node () = incr cur_id; { id = !cur_id; eps = []; trans = []; tag = None } let new_tagged_node tag_op = incr cur_id; { id = !cur_id; eps = []; trans = []; tag = Some tag_op } let seq r1 r2 succ = r1 (r2 succ) (* [is_chars final node] tests whether [node] is a simple character-set node: no epsilon edges, a single transition to [final], and no tag. Used by [alt] to merge adjacent character classes into a single [chars] node instead of introducing an epsilon fork. *) let is_chars final = function | { eps = []; trans = [(c, f)]; tag = None; _ } when f == final -> Some c | _ -> None let chars c succ = let n = new_node () in n.trans <- [(c, succ)]; n let alt r1 r2 succ = let nr1 = r1 succ and nr2 = r2 succ in match (is_chars succ nr1, is_chars succ nr2) with | Some c1, Some c2 -> chars (Cset.union c1 c2) succ | _ -> let n = new_node () in n.eps <- [nr1; nr2]; n let rep r succ = let n = new_node () in n.eps <- [r n; succ]; n (* [r, Star r] with the body duplicated. A single loop entered at the body would put the exit ahead of a second iteration when the first one consumed nothing. *) let plus r succ = r (rep r succ) let eps succ = succ (* eps for epsilon *) let rec repeat r n m succ = assert (0 <= n && n <= m); match (n, m) with | 0, 0 -> succ (* Taking an iteration comes first: repetition is greedy. *) | 0, m -> alt (fun succ -> r (repeat r 0 (m - 1) succ)) eps succ | n, m -> r (repeat r (n - 1) (m - 1) succ) let compl r = let n = new_node () in match is_chars n (r n) with | Some c -> Some (chars (Cset.difference Cset.any c)) | _ -> None let pair_op f r0 r1 = (* Construct subtract or intersection *) let n = new_node () in let to_chars r = is_chars n (r n) in match (to_chars r0, to_chars r1) with | Some c0, Some c1 -> Some (chars (f c0 c1)) | _ -> None let subtract = pair_op Cset.difference let intersection = pair_op Cset.intersection (* Tags for as-bindings *) let cur_tag = ref 0 let reset_tags () = cur_tag := 0 let new_tag () = let t = !cur_tag in incr cur_tag; t let bind r = let start_tag = new_tag () in let end_tag = new_tag () in let wrapped succ = let end_node = new_tagged_node (Set_position { dst = end_tag }) in end_node.eps <- [succ]; let inner = r end_node in let start_node = new_tagged_node (Set_position { dst = start_tag }) in start_node.eps <- [inner]; start_node in (wrapped, start_tag, end_tag) let bind_start_only r = let start_tag = new_tag () in let wrapped succ = let inner = r succ in let start_node = new_tagged_node (Set_position { dst = start_tag }) in start_node.eps <- [inner]; start_node in (wrapped, start_tag) let bind_end_only r = let end_tag = new_tag () in let wrapped succ = let end_node = new_tagged_node (Set_position { dst = end_tag }) in end_node.eps <- [succ]; r end_node in (wrapped, end_tag) let new_disc_cell () = new_tag () let bind_disc r cell value = let wrapped succ = let disc_node = new_tagged_node (Set_value { dst = cell; value }) in disc_node.eps <- [succ]; r disc_node in wrapped (* One rule of a [match%sedlex]: the entry and final nodes of its NFA. *) type rule = { entry : node; final : node } (* [compile_re re] instantiates a regexp by creating a fresh final node and passing it as the successor. *) let compile_re (re : regexp) : rule = let final = new_node () in { entry = re final; final } (* The compiled automaton *) type accept = { rule : int; final_ops : tag_op list } type dfa_state = { trans : (Cset.t * int * tag_op list) array; accept : accept option; } type dfa = dfa_state array type compiled = { dfa : dfa; num_tags : int } let op_dest = function | Copy { dst; _ } | Set_position { dst } | Set_value { dst; _ } -> dst (* Determinization (tagged subset construction, see the overview above) *) (* A memory cell, by index. *) type cell = int module TagMap = Map.Make (Int) module CellMap = Map.Make (Int) (* What a transition writes: the current position or a discriminator. *) type write = Position | Value of int let op_of_write (dst : cell) : write -> tag_op = function | Position -> Set_position { dst } | Value value -> Set_value { dst; value } (* Where a tag's value lives while a transition is computed: in a cell, or in a write of that transition. Same tag and same write: same register. *) type addr = Cell of cell | Pending of write (* One NFA path: the node it reached, where each tag it wrote lives, and the closure's writes, [delayed] until it takes a character transition. *) type 'a config = { node : node; tags : 'a TagMap.t; delayed : write TagMap.t } (* A DFA state: configurations in priority order. A [candidate] comes out of [eps_closure]; a [stored] state has every tag in a cell. *) type 'a state = 'a config list type candidate = addr state type stored = cell state (* Only nodes with character transitions and final nodes belong in a state; epsilon-only nodes would just bloat its key. *) let is_relevant (node : node) : bool = node.trans <> [] || node.eps = [] (* DFS along [eps] lists; the first path to reach a node wins, which is the leftmost-greedy policy. Tag writes on the way are [delayed]. *) let eps_closure (seeds : addr config list) : candidate = let visited = Hashtbl.create 16 in let acc = ref [] in let rec visit node tags delayed = if not (Hashtbl.mem visited node.id) then ( Hashtbl.add visited node.id (); let delayed = match node.tag with | None -> delayed | Some (Set_position { dst }) -> TagMap.add dst Position delayed | Some (Set_value { dst; value }) -> TagMap.add dst (Value value) delayed | Some (Copy _) -> assert false (* never carried by NFA nodes *) in if is_relevant node then acc := { node; tags; delayed } :: !acc; List.iter (fun n -> visit n tags delayed) node.eps) in List.iter (fun c -> visit c.node c.tags c.delayed) seeds; List.rev !acc (* Splits the moves into pairwise-disjoint character sets. Moves come in priority order and a configuration is appended last to each piece it overlaps, so seeds stay in priority order. *) let split_moves (moves : (Cset.t * addr config) list) : (Cset.t * addr config list) list = let add_move pieces (cset, cfg) = let rec insert cset pieces = if Cset.is_empty cset then pieces else ( match pieces with | [] -> [(cset, [cfg])] | (piece, seeds) :: rest -> let inter = Cset.intersection piece cset in if Cset.is_empty inter then (piece, seeds) :: insert cset rest else ( let piece_only = Cset.difference piece inter in let cset_rest = Cset.difference cset inter in let with_cfg = (inter, seeds @ [cfg]) in if Cset.is_empty piece_only then with_cfg :: insert cset_rest rest else (piece_only, seeds) :: with_cfg :: insert cset_rest rest)) in insert cset pieces in List.fold_left add_move [] moves (* Mutable state of the construction *) (* Cell allocation. Canonical cells [0 .. num_logical - 1] (cell = tag id) are only written by final operations; working registers live above. *) module Registers : sig type t val create : num_logical:int -> t (* A fresh working register. *) val alloc : t -> cell (* As opposed to a canonical cell. *) val is_working : t -> cell -> bool (* Canonical and working cells. *) val count : t -> int end = struct type t = { num_logical : int; mutable next_cell : int } let create ~num_logical = { num_logical; next_cell = num_logical } let alloc t = let c = t.next_cell in t.next_cell <- c + 1; c let is_working t c = c >= t.num_logical let count t = t.next_cell end (* State identity modulo register renaming: registers are numbered by first occurrence, so equal keys mean same nodes, same order, same sharing. *) module State_key : sig type t val of_candidate : candidate -> t module Tbl : Hashtbl.S with type key = t end = struct type t = (int * (int * int) list * (int * write) list) list let of_candidate configs = let tbl = Hashtbl.create 8 in let canon tag a = match Hashtbl.find_opt tbl (tag, a) with | Some i -> i | None -> let i = Hashtbl.length tbl in Hashtbl.add tbl (tag, a) i; i in List.map (fun c -> ( c.node.id, List.map (fun (t, a) -> (t, canon t a)) (TagMap.bindings c.tags), TagMap.bindings c.delayed )) configs module Tbl = Hashtbl.Make (struct type nonrec t = t let equal = ( = ) let hash = Hashtbl.hash end) end (* DFA states discovered so far, numbered in creation order. *) type state_table = { by_key : int State_key.Tbl.t; configs : (int, stored) Hashtbl.t; defs : (int, dfa_state) Hashtbl.t; mutable n_states : int; } let make_state_table () : state_table = { by_key = State_key.Tbl.create 31; configs = Hashtbl.create 31; defs = Hashtbl.create 31; n_states = 0; } (* Registers a new state and returns its number. *) let add_state (tbl : state_table) (key : State_key.t) (configs : stored) : int = let num = tbl.n_states in tbl.n_states <- num + 1; State_key.Tbl.add tbl.by_key key num; Hashtbl.add tbl.configs num configs; num type ctx = { regs : Registers.t; rules : rule array; tbl : state_table } (* New state: each [Pending] write gets a fresh cell. Returns the stored state and the Set operations of the transition reaching it. *) let assign_cells (regs : Registers.t) (candidate : candidate) : stored * tag_op list = let assigned = Hashtbl.create 4 in let ops = ref [] in let cell_for_new tag w = match Hashtbl.find_opt assigned (tag, w) with | Some c -> c | None -> let c = Registers.alloc regs in Hashtbl.add assigned (tag, w) c; ops := op_of_write c w :: !ops; c in let configs = List.map (fun c -> { c with tags = TagMap.mapi (fun tag a -> match a with | Cell cell -> cell | Pending w -> cell_for_new tag w) c.tags; }) candidate in (configs, !ops) (* Moves realigning [candidate]'s registers with those of the equal-keyed [existing] state. A parallel move. *) let register_moves (candidate : candidate) (existing : stored) : tag_op list = let moves = ref CellMap.empty in let set_move dst op = match CellMap.find_opt dst !moves with | None -> moves := CellMap.add dst op !moves | Some op' -> assert (op = op') in List.iter2 (fun cand ex -> TagMap.iter (fun tag a -> let dst = TagMap.find tag ex.tags in match a with | Cell src -> if src <> dst then set_move dst (Copy { dst; src }) | Pending w -> set_move dst (op_of_write dst w)) cand.tags) candidate existing; List.map snd (CellMap.bindings !moves) (* Materializes the accepting configuration into the canonical cells: copies of its registers, plus its delayed writes. *) let final_ops_of (regs : Registers.t) (accepting : cell config) : tag_op list = let copies = TagMap.fold (fun tag cell acc -> if cell = tag || TagMap.mem tag accepting.delayed then acc else ( assert (Registers.is_working regs cell); Copy { dst = tag; src = cell } :: acc)) accepting.tags [] in TagMap.fold (fun tag w acc -> op_of_write tag w :: acc) accepting.delayed copies (* The accepting rule is the lowest-numbered one whose final node is in the state (first-match semantics), with its final operations. *) let accept_of (ctx : ctx) (configs : stored) : accept option = let n = Array.length ctx.rules in let rec first_accepting rule = if rule = n then None else ( let final = ctx.rules.(rule).final in match List.find_opt (fun c -> c.node == final) configs with | Some accepting -> Some { rule; final_ops = final_ops_of ctx.regs accepting } | None -> first_accepting (rule + 1)) in first_accepting 0 (* The number of [candidate]'s state, built depth first if new, and the operations of the transition reaching it. *) let rec find_or_add_state (ctx : ctx) (candidate : candidate) : int * tag_op list = let key = State_key.of_candidate candidate in match State_key.Tbl.find_opt ctx.tbl.by_key key with | Some num -> (num, register_moves candidate (Hashtbl.find ctx.tbl.configs num)) | None -> let configs, ops = assign_cells ctx.regs candidate in let num = add_state ctx.tbl key configs in build_state ctx num configs; (num, ops) (* Computes and stores the definition of state [num]. *) and build_state (ctx : ctx) (num : int) (configs : stored) : unit = let trans = transitions ctx configs in let accept = accept_of ctx configs in Hashtbl.add ctx.tbl.defs num { trans; accept } (* Outgoing transitions: split the character moves into disjoint sets, close each piece, and look the pieces up in character-set order, so that state numbers follow that order. End of input is apart. It does not advance the position, but it counts as one more symbol read: a match that reads it is longer than the state's own accept, so any configuration may read it. End of input is read once, so what they reach is cut down to its accepting configuration, which has no transitions, or dropped if it has none: reading end of input then fails like any missing transition. *) and transitions (ctx : ctx) (configs : stored) : (Cset.t * int * tag_op list) array = let is_final c = Array.exists (fun r -> r.final == c.node) ctx.rules in let moves_of ~eof c = (* Taking a transition performs the delayed writes. *) let tags = TagMap.fold (fun tag w tags -> TagMap.add tag (Pending w) tags) c.delayed (TagMap.map (fun cell -> Cell cell) c.tags) in List.filter_map (fun (cset, node) -> let cset = if eof then Cset.intersection cset Cset.eof else Cset.difference cset Cset.eof in if Cset.is_empty cset then None else Some (cset, { node; tags; delayed = TagMap.empty })) c.node.trans in let on_eof = let seeds = List.concat (List.map (moves_of ~eof:true) configs) in match List.find_opt is_final (eps_closure (List.map snd seeds)) with | Some accepting -> [(Cset.eof, [accepting])] | None -> [] in let on_chars = List.map (fun (cset, seeds) -> (cset, eps_closure seeds)) (split_moves (List.concat (List.map (moves_of ~eof:false) configs))) in let pieces = Array.of_list (on_eof @ on_chars) in Array.sort (fun (c1, _) (c2, _) -> compare c1 c2) pieces; Array.map (fun (cset, candidate) -> let num, ops = find_or_add_state ctx candidate in (cset, num, ops)) pieces (* See the implementation overview at the top of this file. *) let compile (rs : regexp array) : compiled = let rules = Array.map compile_re rs in let ctx = { regs = Registers.create ~num_logical:!cur_tag; rules; tbl = make_state_table (); } in let seeds = List.map (fun r -> { node = r.entry; tags = TagMap.empty; delayed = TagMap.empty }) (Array.to_list rules) in let num0, ops = find_or_add_state ctx (eps_closure seeds) in assert (num0 = 0); (* The writes of the initial closure are delayed like any other. *) assert (ops = []); { dfa = Array.init ctx.tbl.n_states (Hashtbl.find ctx.tbl.defs); num_tags = Registers.count ctx.regs; } (* High-level compilation from IR. [compile_ir] lowers [Ir.t] patterns into low-level regexps with tag annotations, then compiles them via [compile]. The lowering phase decides how to allocate tags for [as] bindings: - [Start_plus n]: the position is [n] code points from the token start. - [End_minus n]: the position is [n] code points before the token end. - [Tag {tag; offset}]: read memory cell [tag] and add [offset]. When both boundaries of a capture can be expressed as [Start_plus] or [End_minus], no memory cells are needed at all. Or-patterns [(p1 as x) | (p2 as x)] additionally use discriminator cells: integer values that record which branch was taken, so the code generator can emit the correct position extraction at match time. *) type pos_expr = | Tag of { tag : int; offset : int } | Start_plus of int | End_minus of int type compiled_binding = { name : string; start_pos : pos_expr; end_pos : pos_expr; disc : (int * int) list; } type compiled_ir = { dfa : dfa; num_tags : int; bindings : compiled_binding list array; } (* [shift_pos pe delta] shifts a position expression by [delta] code points (positive = forward, negative = backward). Returns [None] if either argument is unknown. *) let shift_pos pe delta = match (pe, delta) with | Some (Start_plus n), Some d -> Some (Start_plus (n + d)) | Some (End_minus n), Some d -> Some (End_minus (n - d)) | Some (Tag { tag; offset }), Some d -> Some (Tag { tag; offset = offset + d }) | _ -> None let advance pe len = shift_pos pe len let retreat pe len = shift_pos pe (Option.map Int.neg len) (* [add_discriminators branches] takes a list of [(regexp, bindings)] pairs from an n-ary alternation and wraps each branch with a discriminator tag so the generated code can tell which branch matched. Branches with identical bindings share the same discriminator value. If all branches have identical bindings, no discriminator cell is allocated. *) let add_discriminators (branches : (regexp * compiled_binding list) list) = let fold_alt = function | [] -> assert false | (r, _) :: rest -> List.fold_left (fun acc (r, _) -> alt acc r) r rest in (* Check if all branches produce identical bindings — if so, no discriminator is needed at all. *) let all_same = match branches with | [] | [_] -> true | (_, first) :: rest -> List.for_all (fun (_, tags) -> tags = first) rest in if all_same then (fold_alt branches, snd (List.hd branches)) else ( let disc_cell = new_disc_cell () in let stamp value tags = List.map (fun (ti : compiled_binding) -> { ti with disc = (disc_cell, value) :: ti.disc }) tags in (* Assign discriminator values. Branches with identical bindings share the same value. *) let next_val = ref 0 in let seen : (compiled_binding list * int) list ref = ref [] in let get_value tags = match List.assoc_opt tags !seen with | Some v -> v | None -> let v = !next_val in incr next_val; seen := (tags, v) :: !seen; v in let wrapped = List.map (fun (r, tags) -> let v = get_value tags in (bind_disc r disc_cell v, stamp v tags)) branches in (fold_alt wrapped, List.concat_map snd wrapped)) (* [lower ir ~left ~right] converts an IR pattern to a low-level regexp and a list of compiled bindings. [left] and [right] are the known position contexts at the start and end of this pattern element. *) let rec lower ~left ~right (ir : Ir.t) : regexp * compiled_binding list = match ir with | Ir.Chars cset -> (chars cset, []) | Ir.Eps -> (eps, []) | Ir.Star inner -> let r, _ = lower ~left:None ~right:None inner in (rep r, []) | Ir.Plus inner -> let r, _ = lower ~left:None ~right:None inner in (plus r, []) | Ir.Rep (inner, n, m) -> let r, _ = lower ~left:None ~right:None inner in (repeat r n m, []) | Ir.Capture (name, inner) -> (* Named capture — try to derive each boundary from [left]/[right] context or [fixed_length]; allocate tags only for boundaries that cannot be computed statically. Best case: 0 tags. Worst case: 2. *) let r, tags = lower ~left ~right inner in let elem_len = Ir.fixed_length inner in let known_start = match left with Some _ -> left | None -> retreat right elem_len in let known_end = match right with | Some _ -> right | None -> advance known_start elem_len in let st, et, r = match (known_start, known_end) with | Some st, Some et -> (st, et, r) | Some st, None -> let wrapped, end_tag = bind_end_only r in (st, Tag { tag = end_tag; offset = 0 }, wrapped) | None, Some et -> let wrapped, start_tag = bind_start_only r in (Tag { tag = start_tag; offset = 0 }, et, wrapped) | None, None -> ( match elem_len with | Some len -> let wrapped, start_tag = bind_start_only r in ( Tag { tag = start_tag; offset = 0 }, Tag { tag = start_tag; offset = len }, wrapped ) | None -> let wrapped, start_tag, end_tag = bind r in ( Tag { tag = start_tag; offset = 0 }, Tag { tag = end_tag; offset = 0 }, wrapped )) in (r, { name; start_pos = st; end_pos = et; disc = [] } :: tags) | Ir.Alt branches -> let lowered = List.map (lower ~left ~right) branches in add_discriminators lowered | Ir.Seq elems -> (* Sequence — propagate left/right position contexts through elements. Right positions are computed right-to-left; left positions are updated left-to-right after lowering each element. *) let _, rights = List.fold_right (fun e (acc, l) -> (retreat acc (Ir.fixed_length e), acc :: l)) elems (right, []) in (* Fallback for the left context: if [advance] returns [None] (because the current left is unknown or the element has variable length), but the element was a [Capture] whose end position is a [Tag], we can use that tag as the [left] anchor for the next element — it records a runtime position. [Start_plus]/[End_minus] endpoints don't help here: they are already factored into [advance], so if [advance] failed, they have nothing more to offer. *) let left_from_end_tag ir tags' = match ir with | Ir.Capture _ -> ( match tags' with | { end_pos = Tag _ as et; _ } :: _ -> Some et | _ -> None) | _ -> None in (* [seq] is function composition and [eps] its identity, so the fold needs no special case for the first element. *) let _, r_acc, tags_acc = List.fold_left2 (fun (cur_left, r_acc, tags_acc) e right -> let r', tags' = lower ~left:cur_left ~right e in let new_left = match advance cur_left (Ir.fixed_length e) with | Some _ as s -> s | None -> left_from_end_tag e tags' in (new_left, seq r_acc r', tags_acc @ tags')) (left, eps, []) elems rights in (r_acc, tags_acc) let compile_ir (rules : Ir.t array) = Array.iter (fun ir -> Ir.check_invariant ir) rules; reset_tags (); let lowered = Array.map (fun ir -> lower ~left:(Some (Start_plus 0)) ~right:(Some (End_minus 0)) ir) rules in let regexps = Array.map fst lowered in let bindings = Array.map snd lowered in let compiled = compile regexps in { dfa = compiled.dfa; num_tags = compiled.num_tags; bindings } let cset_to_label cset = let escape_dot c = match c with | '"' -> "\\\"" | '\\' -> "\\\\" | '<' -> "\\<" | '>' -> "\\>" | _ -> String.make 1 c in let format_interval (lo, hi) = if lo = -1 && hi = -1 then "EOF" else if lo = hi then if lo >= 32 && lo <= 126 then "'" ^ escape_dot (Char.chr lo) ^ "'" else Printf.sprintf "U+%04X" lo else if lo >= 32 && lo <= 126 && hi >= 32 && hi <= 126 then "'" ^ escape_dot (Char.chr lo) ^ "'-'" ^ escape_dot (Char.chr hi) ^ "'" else Printf.sprintf "U+%04X-U+%04X" lo hi in String.concat ", " (List.map format_interval (cset : Cset.t :> (int * int) list)) let tag_op_to_string = function | Set_position { dst } -> "t" ^ string_of_int dst | Set_value { dst; value } -> "d" ^ string_of_int dst ^ "=" ^ string_of_int value | Copy { dst; src } -> "t" ^ string_of_int dst ^ "<-t" ^ string_of_int src let dfa_to_dot dfa = let buf = Buffer.create 1024 in let bprintf = Printf.bprintf in bprintf buf "digraph {\n"; bprintf buf " rankdir=LR;\n"; bprintf buf " node [shape=circle];\n\n"; bprintf buf " _start [shape=point];\n"; bprintf buf " _start -> state0;\n\n"; Array.iteri (fun i { trans; accept } -> (match accept with | None -> bprintf buf " state%d [label=\"%d\"];\n" i i | Some { rule; final_ops } -> let ops = if final_ops = [] then "" else "\\n{" ^ String.concat "," (List.map tag_op_to_string final_ops) ^ "}" in bprintf buf " state%d [label=\"%d\\n[rule %d]%s\", shape=doublecircle];\n" i i rule ops); Array.iter (fun (cset, target, tags) -> let label = cset_to_label cset in let label = if tags = [] then label else label ^ " {" ^ String.concat "," (List.map tag_op_to_string tags) ^ "}" in bprintf buf " state%d -> state%d [label=\"%s\"];\n" i target label) trans) dfa; bprintf buf "}\n"; Buffer.contents buf sedlex-3.8.1/src/compiler/sedlex.mli000066400000000000000000000167351526113672700174250ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (** {2 Regexp combinators} Regular expressions are built from combinators and compiled to a DFA. *) (** Abstract type of regular expressions. *) type regexp (** [chars cset] matches a single code point in [cset]. *) val chars : Cset.t -> regexp (** [seq r1 r2] matches [r1] followed by [r2] (concatenation). *) val seq : regexp -> regexp -> regexp (** [alt r1 r2] matches [r1] or [r2] (alternation). When both operands are simple [chars] regexps, their character sets are merged into one. *) val alt : regexp -> regexp -> regexp (** [rep r] matches zero or more repetitions of [r] (Kleene star). *) val rep : regexp -> regexp (** [plus r] matches one or more repetitions of [r]. *) val plus : regexp -> regexp (** [repeat r n m] matches between [n] and [m] repetitions of [r] (bounded repetition). Requires [0 <= n <= m]. *) val repeat : regexp -> int -> int -> regexp (** The empty regexp — matches the empty string (epsilon). *) val eps : regexp (** If the argument is a single [chars] regexp, returns a regexp which matches the complement set. Otherwise returns [None]. *) val compl : regexp -> regexp option (** If each argument is a single [chars] regexp, returns a regexp which matches the set (arg1 - arg2). Otherwise returns [None]. *) val subtract : regexp -> regexp -> regexp option (** If each argument is a single [chars] regexp, returns a regexp which matches the intersection set. Otherwise returns [None]. *) val intersection : regexp -> regexp -> regexp option (** {2 Tagged DFA for [as] bindings} Named sub-match bindings (e.g. [Star any as x]) are implemented using tagged transitions in the DFA. Each [as] binding introduces up to two tags that record the start and end positions of the sub-match in the lexbuf's memory cells at runtime. When one boundary can be computed from a known offset, only one tag is needed ([bind_start_only] / [bind_end_only]); when both boundaries are known, no tags are needed at all. Or-patterns [(p1 as x) | (p2 as x)] additionally use discriminator cells: integer values that record which branch was taken, so the PPX can extract the correct positions at match time. *) (** Tag operations emitted on DFA transitions. *) type tag_op = | Set_position of { dst : int } (** [Set_position {dst}]: record the current lexbuf position in memory cell [dst]. *) | Set_value of { dst : int; value : int } (** [Set_value {dst; value}]: record integer [value] in memory cell [dst] (used for or-pattern discriminators). *) | Copy of { dst : int; src : int } (** [Copy {dst; src}]: copy the contents of cell [src] into cell [dst]. Emitted when determinization must preserve a position that a parallel NFA path is about to overwrite. *) (** [op_dest op] is the memory cell written by [op]. *) val op_dest : tag_op -> int (** [bind r] wraps [r] with start/end tag epsilon nodes. Returns [(wrapped_regexp, start_tag, end_tag)] where [start_tag] and [end_tag] are the allocated memory cell indices. *) val bind : regexp -> regexp * int * int (** Allocate a fresh memory cell for an or-pattern discriminator. *) val new_disc_cell : unit -> int (** [bind_disc r cell value] appends an epsilon node that sets [cell] to [value] after [r] matches. Used to tag each branch of an or-pattern so the PPX can tell which branch matched. *) val bind_disc : regexp -> int -> int -> regexp (** Like [bind], but only wraps with a start tag (no end tag). Used when the end position can be computed from a known offset. *) val bind_start_only : regexp -> regexp * int (** Like [bind], but only wraps with an end tag (no start tag). Used when the start position can be computed from a known offset. *) val bind_end_only : regexp -> regexp * int (** Reset the tag counter. Called before compiling each [match%sedlex] block. *) val reset_tags : unit -> unit (** {2 DFA compilation} *) (** What an accepting state accepts. *) type accept = { rule : int; (** The lowest-numbered, hence highest-priority, accepting rule, matching the first-match semantics of [match%sedlex]. *) final_ops : tag_op list; (** Tag operations to execute when entering the state, just before [Sedlexing.mark]: they materialize the accepting configuration into the cells read by the binding extraction code. *) } type dfa_state = { trans : (Cset.t * int * tag_op list) array; (** Each transition: (character set, target state, tag operations to execute when this transition fires). The operations form a parallel move: a [Copy] reads its source as it was before any operation of the same list executed, and no two operations write the same cell. They execute {e before} the character of the transition is consumed. A character set is either [Cset.eof] or does not contain end of input, and the target of a [Cset.eof] transition has no transitions: end of input is read once. *) accept : accept option; (** [None] for non-accepting states. *) } (** DFA states, indexed by state number. State 0 is the initial state. *) type dfa = dfa_state array (** Result of [compile]. *) type compiled = { dfa : dfa; num_tags : int; (** Total number of memory cells needed at runtime. When [num_tags = 0], no memory is allocated (pattern has no [as] bindings). *) } (** [compile rules] determinizes the NFA for an array of regexp rules using subset construction. Returns the DFA and the total number of memory cells needed for [as] bindings. State 0 is always the initial state. *) val compile : regexp array -> compiled (** {2 High-level compilation from IR} [compile_ir] takes an array of {!Ir.t} patterns, handles tag allocation (including [Start_plus]/[End_minus] optimizations), discriminator insertion for or-patterns, and DFA construction. Returns the DFA and all information needed to generate binding extraction code. *) (** How to compute a sub-match boundary position at runtime. *) type pos_expr = | Tag of { tag : int; offset : int } (** Read memory cell [tag] and add [offset]. *) | Start_plus of int (** Position is [n] code points from the token start. *) | End_minus of int (** Position is [n] code points before the token end. *) (** Information about a single binding in the compiled output. *) type compiled_binding = { name : string; start_pos : pos_expr; end_pos : pos_expr; disc : (int * int) list; (** Discriminator conditions: [(cell, value)] pairs. Empty for simple (non-or) bindings. For or-patterns, branches with different positions have distinct values; branches with identical positions share a value. *) } (** Result of [compile_ir]. *) type compiled_ir = { dfa : dfa; num_tags : int; bindings : compiled_binding list array; (** [bindings.(i)] is the list of binding info for rule [i]. *) } (** [compile_ir rules] compiles an array of IR patterns into a tagged DFA. Raises [Assert_failure] if invariant checking fails. *) val compile_ir : Ir.t array -> compiled_ir (** [dfa_to_dot dfa] returns a Graphviz DOT representation of the DFA, including state labels, accepting state markers, transition character sets, and tag operations on edges. *) val dfa_to_dot : dfa -> string sedlex-3.8.1/src/generator/000077500000000000000000000000001526113672700155765ustar00rootroot00000000000000sedlex-3.8.1/src/generator/README.md000066400000000000000000000026251526113672700170620ustar00rootroot00000000000000# Unicode specification extraction The file `src/syntax/unicode.ml` is generated using the data available at [unicode.org](https://www.unicode.org/Public/). The rule with `targets unicode.ml` at `src/syntax/dune` is the main entry point for this process. It specifies how `unicode.ml` should be generated when running `dune @build` and triggers: * download of data files at `src/generator/data` * build of `src/generator/gen_unicode.exe` * generation `unicode.ml` and places a copy in the source tree and a copy in the build tree The rule is ignored when using the `--ignore-promoted-rules` option. This option is also implied when using `-p`/`--for-release-of-packages` which is used for production build so production build do not download the text data and re-generate `unicode.ml`. However, each development build re-generates a `unicode.ml` file which is placed into the source tree and, thus, can be easily commited when it is updated. See: [dune documentation](https://dune.readthedocs.io/en/latest/dune-files.html#modes) for more information. ## Update to new Unicode versions To update the supported version, update the URL at `src/generator/data/base_url`. Make sure to not include a trailing new line so that it is properly read in `src/generator/data/dune`. Finally, place a copy of the old `unicode.ml` at `examples/unicode_old.ml` and update `test_versions` and `regressions` in `examples/regressions.ml`. sedlex-3.8.1/src/generator/data/000077500000000000000000000000001526113672700165075ustar00rootroot00000000000000sedlex-3.8.1/src/generator/data/base_url000066400000000000000000000000451526113672700202250ustar00rootroot00000000000000https://www.unicode.org/Public/18.0.0sedlex-3.8.1/src/generator/data/dune000066400000000000000000000011311526113672700173610ustar00rootroot00000000000000(rule (target DerivedCoreProperties.txt) (deps base_url) (action (run curl -L -s %{read:base_url}/ucd/DerivedCoreProperties.txt -o %{target}))) (rule (target DerivedGeneralCategory.txt) (deps base_url) (action (run curl -L -s %{read:base_url}/ucd/extracted/DerivedGeneralCategory.txt -o %{target}))) (rule (target PropList.txt) (deps base_url) (action (run curl -L -s %{read:base_url}/ucd/PropList.txt -o %{target}))) (rule (target UnicodeData.txt) (deps base_url) (action (run curl -L -s %{read:base_url}/ucd/UnicodeData.txt -o %{target}))) sedlex-3.8.1/src/generator/dune000066400000000000000000000001021526113672700164450ustar00rootroot00000000000000(executable (name gen_unicode) (libraries str sedlex.compiler)) sedlex-3.8.1/src/generator/gen_unicode.ml000066400000000000000000000133551526113672700204160ustar00rootroot00000000000000(* This file generates unicode data from * the files exported at https://www.unicode.org/Public/ * and stored at src/generator/data. *) open Sedlex_compiler module SSet = Set.Make (String) let target = Sys.argv.(1) let categories = Hashtbl.create 1024 let labels = Hashtbl.create 1024 (* Drop comments and split semi-column separated fields *) let parse_line l = let l = match String.index_opt l '#' with None -> l | Some i -> String.sub l 0 i in String.split_on_char ';' l |> List.map String.trim let parse_code s = try int_of_string (Printf.sprintf "0x%s" s) with _ -> failwith (Printf.sprintf "invalid code %s" s) let parse_category x = String.lowercase_ascii (String.trim x) let parse_prop x = String.lowercase_ascii (String.trim x) let parse_interval s = match String.split_on_char '.' (String.trim s) with | [] -> assert false | [x] -> let x = parse_code x in Cset.singleton x | [x; ""; y] -> let x = parse_code x and y = parse_code y in Cset.interval x y | _ -> failwith (Printf.sprintf "invalid interval %s" s) let print_elements ch hashtbl cats = let cats_set = SSet.of_list cats in let all_keys = SSet.of_seq (Hashtbl.to_seq_keys hashtbl) in let missing = SSet.diff cats_set all_keys in let ignoring = SSet.diff all_keys cats_set in let len = List.length cats in List.iter (fun c -> let entries = List.map (fun (b, e) -> Printf.sprintf "0x%x, 0x%x" b e) (Cset.union_list (Hashtbl.find_all hashtbl c) :> (int * int) list) in Printf.fprintf ch " let %s = Cset.of_list\n [" c; List.iteri (fun i x -> if i > 0 then if i mod 5 = 0 then Printf.fprintf ch ";\n " else Printf.fprintf ch "; "; Printf.fprintf ch "%s" x) entries; Printf.fprintf ch "]\n\n") cats; Printf.fprintf ch " let list = [\n"; List.iteri (fun pos c -> Printf.fprintf ch " (%S, %s)%s\n" c c (if pos == len - 1 then "" else ";")) cats; Printf.fprintf ch " ]\n\n"; if not (SSet.is_empty ignoring) then ( Printf.fprintf ch "(* ignoring:\n"; SSet.iter (fun s -> Printf.fprintf ch " - %s\n" s) ignoring; Printf.fprintf ch "*)\n"); if not (SSet.is_empty missing) then ( Printf.fprintf ch "(* missing:\n"; SSet.iter (fun s -> Printf.fprintf ch " - %s\n" s) missing; Printf.fprintf ch "*)\n") let files = [ ( "PropList.txt", fun s -> match parse_line s with | [""] -> () | [interval; prop] -> let interval = parse_interval interval in let prop = parse_prop prop in Hashtbl.add labels prop interval | _ -> assert false ); ( "DerivedCoreProperties.txt", fun s -> match parse_line s with | [""] -> () | [interval; prop] -> let interval = parse_interval interval in let prop = parse_prop prop in Hashtbl.add labels prop interval | [_interval; "InCB"; ("Extend" | "Consonant" | "Linker")] -> (* TODO: support non-binary properties? *) () | _ -> assert false ); ( "DerivedGeneralCategory.txt", fun s -> match parse_line s with | [""] -> () | [interval; cat] -> let interval = parse_interval interval in let cat = parse_category cat in Hashtbl.add categories cat interval | _ -> assert false ); ( "UnicodeData.txt", fun s -> match parse_line s with | [""] -> () | interval :: _ :: cat :: _ -> let interval = parse_interval interval in let cat = parse_category cat in Hashtbl.add categories cat interval | _ -> assert false ); ] let read_version fname = let version_rex = Str.regexp "^# PropList-\\([0-9]+\\.[0-9]+\\.[0-9]+\\)\\.txt" in let ch = open_in_bin fname in let s = input_line ch in close_in ch; ignore (Str.string_match version_rex s 0); Str.matched_group 1 s let exported_categories = [ "cc"; "cf"; "cn"; "co"; "cs"; "ll"; "lm"; "lo"; "lt"; "lu"; "mc"; "me"; "mn"; "nd"; "nl"; "no"; "pc"; "pd"; "pe"; "pf"; "pi"; "po"; "ps"; "sc"; "sk"; "sm"; "so"; "zl"; "zp"; "zs"; ] let exported_properties = [ "alphabetic"; "ascii_hex_digit"; "hex_digit"; "id_continue"; "id_start"; "lowercase"; "math"; "other_alphabetic"; "other_lowercase"; "other_math"; "other_uppercase"; "uppercase"; "white_space"; "xid_continue"; "xid_start"; ] let () = let base_dir = Filename.concat (Filename.dirname Sys.executable_name) "data" in let version = read_version (Filename.concat base_dir "PropList.txt") in List.iter (fun (fname, fn) -> let ch = open_in_bin (Filename.concat base_dir fname) in try while true do let ret = input_line ch in fn ret done with End_of_file -> close_in ch) files; let ch = open_out_bin target in Printf.fprintf ch {|[@@@ocamlformat "disable"]|}; Printf.fprintf ch "\n\n"; Printf.fprintf ch "(* This file was automatically generated, do not edit. *)\n"; Printf.fprintf ch "(* Edit gen_unicode.ml.inc instead. *)\n\n"; Printf.fprintf ch "open Sedlex_compiler\n\n"; Printf.fprintf ch "let version = %S\n\n" version; Printf.fprintf ch "module Categories = struct\n\n"; print_elements ch categories exported_categories; Printf.fprintf ch "end\n\n"; Printf.fprintf ch "module Properties = struct\n\n"; print_elements ch labels exported_properties; Printf.fprintf ch "end\n"; close_out ch sedlex-3.8.1/src/lib/000077500000000000000000000000001526113672700143565ustar00rootroot00000000000000sedlex-3.8.1/src/lib/dune000066400000000000000000000001211526113672700152260ustar00rootroot00000000000000(library (name sedlex) (public_name sedlex) (wrapped false) (libraries gen)) sedlex-3.8.1/src/lib/sedlexing.ml000066400000000000000000000713341526113672700167020ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) exception InvalidCodepoint of int exception MalFormed module Uchar = struct include Uchar let of_int x = if Uchar.is_valid x then Uchar.unsafe_of_int x else raise MalFormed (* As in the standard library since OCaml 4.14, redefined here so that they are inlined. *) let[@inline] utf_8_byte_length u = let c = to_int u in if c < 0x80 then 1 else if c < 0x800 then 2 else if c < 0x10000 then 3 else 4 let[@inline] utf_16_byte_length u = if to_int u < 0x10000 then 2 else 4 end module Bytes = struct include Bytes (* [set_utf_8_uchar], [set_utf_16be_uchar] and [set_utf_16le_uchar] as in the standard library since OCaml 4.14, redefined here so that they are inlined: they write the encoding of a code point at an index and return its length, or 0 without writing anything when it does not fit. *) let[@inline] unsafe_set_uint8 b i x = unsafe_set b i (Char.unsafe_chr x) let[@inline] set_utf_8_uchar b i u = let c = Uchar.to_int u in let room = length b - i in if i < 0 || room <= 0 then raise (Invalid_argument "index out of bounds") else if c < 0x80 then begin unsafe_set_uint8 b i c; 1 end else if c < 0x800 then if room < 2 then 0 else begin unsafe_set_uint8 b i (0xC0 lor (c lsr 6)); unsafe_set_uint8 b (i + 1) (0x80 lor (c land 0x3F)); 2 end else if c < 0x10000 then if room < 3 then 0 else begin unsafe_set_uint8 b i (0xE0 lor (c lsr 12)); unsafe_set_uint8 b (i + 1) (0x80 lor ((c lsr 6) land 0x3F)); unsafe_set_uint8 b (i + 2) (0x80 lor (c land 0x3F)); 3 end else if room < 4 then 0 else begin unsafe_set_uint8 b i (0xF0 lor (c lsr 18)); unsafe_set_uint8 b (i + 1) (0x80 lor ((c lsr 12) land 0x3F)); unsafe_set_uint8 b (i + 2) (0x80 lor ((c lsr 6) land 0x3F)); unsafe_set_uint8 b (i + 3) (0x80 lor (c land 0x3F)); 4 end (* [hi] and [lo] are the offsets of the high and low bytes of a 16-bit unit, which give the byte order. *) let[@inline] unsafe_set_uint16 ~hi ~lo b i x = unsafe_set_uint8 b (i + hi) (x lsr 8); unsafe_set_uint8 b (i + lo) (x land 0xFF) let[@inline] set_utf_16_uchar ~hi ~lo b i u = let c = Uchar.to_int u in let room = length b - i in if i < 0 || room <= 0 then raise (Invalid_argument "index out of bounds") else if c < 0x10000 then if room < 2 then 0 else begin unsafe_set_uint16 ~hi ~lo b i c; 2 end else if room < 4 then 0 else begin (* a surrogate pair *) let c = c - 0x10000 in unsafe_set_uint16 ~hi ~lo b i (0xD800 lor (c lsr 10)); unsafe_set_uint16 ~hi ~lo b (i + 2) (0xDC00 lor (c land 0x3FF)); 4 end let[@inline] set_utf_16be_uchar b i u = set_utf_16_uchar ~hi:0 ~lo:1 b i u let[@inline] set_utf_16le_uchar b i u = set_utf_16_uchar ~hi:1 ~lo:0 b i u end (* shadow polymorphic equal *) let ( = ) (a : int) b = a = b let ( >>| ) o f = match o with Some x -> Some (f x) | None -> None (* Absolute position from the beginning of the stream *) type apos = int type lexbuf = { refill : Uchar.t array -> int -> int -> int; bytes_per_char : Uchar.t -> int; mutable buf : Uchar.t array; (* Number of valid uchars in [buf] (from index 0 to len-1). *) mutable len : int; (* Cumulative uchar count: number of uchars discarded before buf[0]. Absolute uchar position of buf[i] = offset + i. *) mutable offset : apos; (* Cumulative byte count: number of bytes discarded before buf[0]. *) mutable bytes_offset : apos; (* Current read position in [buf] (buffer-relative index, 0-based). *) mutable pos : int; (* Current read position in bytes (buffer-relative). *) mutable bytes_pos : int; (* Absolute position of the beginning of the current line, in uchar. *) mutable curr_bol : int; (* Absolute position of the beginning of the current line, in bytes. *) mutable curr_bytes_bol : int; (* Index of the current line in the input stream. *) mutable curr_line : int; (* Token start position in [buf], in uchars (buffer-relative). *) mutable start_pos : int; (* Token start position in bytes (buffer-relative). *) mutable start_bytes_pos : int; (* Absolute beginning-of-line position at token start, in uchars. *) mutable start_bol : int; (* Absolute beginning-of-line position at token start, in bytes. *) mutable start_bytes_bol : int; (* Line number at token start (starts from 1). *) mutable start_line : int; (* Backtrack snapshot: saved by [mark], restored by [backtrack]. *) mutable marked_pos : int; mutable marked_bytes_pos : int; mutable marked_bol : int; mutable marked_bytes_bol : int; mutable marked_line : int; (* The rule index stored by [mark]. *) mutable marked_val : int; mutable filename : string; (* True when the input source is exhausted. *) mutable finished : bool; (* Memory cells for tagged DFA transitions (as-bindings). A single int array stores both positions and discriminator values, distinguished by range: - positions: buffer-relative uchar indices (>= 0), adjusted by [refill] when the buffer is compacted, and converted to token-relative offsets on read by [__private__mem_pos]. - discriminator values: stored as [-(v + 2)], always <= -2, disjoint from positions. The cells are not cleared between tokens: an action only reads cells written on the path of its match. [mark] and [backtrack] leave the cells alone: those read after a match are only written on entering an accepting state, just before [mark]. *) mutable __private__mem : int array; } let chunk_size = 512 let empty_lexbuf bytes_per_char = { refill = (fun _ _ _ -> assert false); bytes_per_char; buf = [||]; len = 0; offset = 0; bytes_offset = 0; pos = 0; bytes_pos = 0; curr_bol = 0; curr_bytes_bol = 0; curr_line = 1; start_pos = 0; start_bytes_pos = 0; start_bol = 0; start_bytes_bol = 0; start_line = 0; marked_pos = 0; marked_bytes_pos = 0; marked_bol = 0; marked_bytes_bol = 0; marked_line = 0; marked_val = 0; filename = ""; finished = false; __private__mem = [||]; } let dummy_uchar = Uchar.of_int 0 let nl_uchar = Uchar.of_int 10 let create ?(bytes_per_char = fun _ -> 1) refill = { (empty_lexbuf bytes_per_char) with refill; buf = Array.make chunk_size dummy_uchar; } let set_position ?bytes_position lexbuf position = lexbuf.offset <- position.Lexing.pos_cnum - lexbuf.pos; lexbuf.curr_bol <- position.Lexing.pos_bol; lexbuf.curr_line <- position.Lexing.pos_lnum; let bytes_position = Option.value ~default:position bytes_position in lexbuf.bytes_offset <- bytes_position.Lexing.pos_cnum - lexbuf.bytes_pos; lexbuf.curr_bytes_bol <- bytes_position.Lexing.pos_bol let set_filename lexbuf fname = lexbuf.filename <- fname let from_gen ?bytes_per_char gen = let malformed = ref false in let refill buf pos len = let rec loop i = if !malformed then raise MalFormed; if i >= len then len else ( match gen () with | Some c -> buf.(pos + i) <- c; loop (i + 1) | None -> i | exception MalFormed when i <> 0 -> malformed := true; i) in loop 0 in create ?bytes_per_char refill let from_int_array ?bytes_per_char a = from_gen ?bytes_per_char (Gen.init ~limit:(Array.length a) (fun i -> Uchar.of_int a.(i))) let from_uchar_array ?(bytes_per_char = fun _ -> 1) a = let len = Array.length a in { (empty_lexbuf bytes_per_char) with buf = Array.init len (fun i -> a.(i)); len; finished = true; } let refill lexbuf = if lexbuf.len + chunk_size > Array.length lexbuf.buf then begin let s = lexbuf.start_pos in let s_bytes = lexbuf.start_bytes_pos in let ls = lexbuf.len - s in if ls + chunk_size <= Array.length lexbuf.buf then Array.blit lexbuf.buf s lexbuf.buf 0 ls else begin let newlen = (Array.length lexbuf.buf + chunk_size) * 2 in let newbuf = Array.make newlen dummy_uchar in Array.blit lexbuf.buf s newbuf 0 ls; lexbuf.buf <- newbuf end; lexbuf.len <- ls; lexbuf.offset <- lexbuf.offset + s; lexbuf.bytes_offset <- lexbuf.bytes_offset + s_bytes; lexbuf.pos <- lexbuf.pos - s; lexbuf.bytes_pos <- lexbuf.bytes_pos - s_bytes; lexbuf.marked_pos <- lexbuf.marked_pos - s; lexbuf.marked_bytes_pos <- lexbuf.marked_bytes_pos - s_bytes; lexbuf.start_pos <- 0; lexbuf.start_bytes_pos <- 0; (* Adjust tagged DFA memory cells: position cells (>= 0) are buffer-relative uchar indices and must be shifted by [s] after compaction. Value cells (<= -2) are left unchanged. Cells left over from an earlier token may be shifted too, which is harmless. *) for i = 0 to Array.length lexbuf.__private__mem - 1 do if lexbuf.__private__mem.(i) >= 0 then lexbuf.__private__mem.(i) <- lexbuf.__private__mem.(i) - s done end; let n = lexbuf.refill lexbuf.buf lexbuf.pos chunk_size in if n = 0 then lexbuf.finished <- true else lexbuf.len <- lexbuf.len + n let new_line lexbuf = lexbuf.curr_line <- lexbuf.curr_line + 1; lexbuf.curr_bol <- lexbuf.pos + lexbuf.offset; lexbuf.curr_bytes_bol <- lexbuf.bytes_pos + lexbuf.bytes_offset let[@inline always] next_aux some none lexbuf = if (not lexbuf.finished) && lexbuf.pos = lexbuf.len then refill lexbuf; if lexbuf.finished && lexbuf.pos = lexbuf.len then none else begin let ret = lexbuf.buf.(lexbuf.pos) in lexbuf.pos <- lexbuf.pos + 1; lexbuf.bytes_pos <- lexbuf.bytes_pos + lexbuf.bytes_per_char ret; if Uchar.equal ret nl_uchar then new_line lexbuf; some ret end let next lexbuf = (next_aux [@inlined]) (fun x -> Some x) None lexbuf let __private__next_int lexbuf = (next_aux [@inlined]) Uchar.to_int (-1) lexbuf let mark lexbuf i = lexbuf.marked_pos <- lexbuf.pos; lexbuf.marked_bytes_pos <- lexbuf.bytes_pos; lexbuf.marked_bol <- lexbuf.curr_bol; lexbuf.marked_bytes_bol <- lexbuf.curr_bytes_bol; lexbuf.marked_line <- lexbuf.curr_line; lexbuf.marked_val <- i let start lexbuf = lexbuf.start_pos <- lexbuf.pos; lexbuf.start_bytes_pos <- lexbuf.bytes_pos; lexbuf.start_bol <- lexbuf.curr_bol; lexbuf.start_bytes_bol <- lexbuf.curr_bytes_bol; lexbuf.start_line <- lexbuf.curr_line; mark lexbuf (-1) let backtrack lexbuf = lexbuf.pos <- lexbuf.marked_pos; lexbuf.bytes_pos <- lexbuf.marked_bytes_pos; lexbuf.curr_bol <- lexbuf.marked_bol; lexbuf.curr_bytes_bol <- lexbuf.marked_bytes_bol; lexbuf.curr_line <- lexbuf.marked_line; lexbuf.marked_val let rollback lexbuf = lexbuf.pos <- lexbuf.start_pos; lexbuf.bytes_pos <- lexbuf.start_bytes_pos; lexbuf.curr_bol <- lexbuf.start_bol; lexbuf.curr_bytes_bol <- lexbuf.start_bytes_bol; lexbuf.curr_line <- lexbuf.start_line (* Tagged DFA memory cells for `as` bindings. Positions are stored as buffer-relative uchar indices (>= 0), converted to token-relative offsets on read by [__private__mem_pos]. Discriminator values are stored as -(v + 2), always <= -2. This range convention lets [refill] adjust only position cells (>= 0) when compacting the buffer. *) (* Only grows the array. The cells keep the contents of the previous token: clearing them at each token costs more than the tag operations. *) let __private__ensure_mem lexbuf n = if Array.length lexbuf.__private__mem < n then (* -1 is neither a position nor a value. Nothing relies on it: reused cells hold whatever earlier tokens left. *) lexbuf.__private__mem <- Array.make n (-1) let __private__set_mem_pos lexbuf i = lexbuf.__private__mem.(i) <- lexbuf.pos let __private__set_mem_prev_pos lexbuf i = lexbuf.__private__mem.(i) <- lexbuf.pos - 1 let __private__set_mem_value lexbuf i v = assert (v >= 0); lexbuf.__private__mem.(i) <- -(v + 2) (* Copies the raw cell contents, preserving the position/value encoding. *) let __private__copy_mem lexbuf dst src = lexbuf.__private__mem.(dst) <- lexbuf.__private__mem.(src) (* Raw cell access, used by generated code to save a cell in a local variable when a parallel register move both reads and overwrites it. The value is opaque (position/value encoding preserved); [__private__mem_set] must only be given values obtained from [__private__mem_get]. *) let __private__mem_get lexbuf i = lexbuf.__private__mem.(i) let __private__mem_set lexbuf i v = lexbuf.__private__mem.(i) <- v (* Returns position relative to token start, for use in sub_lexeme. *) let __private__mem_pos lexbuf i = lexbuf.__private__mem.(i) - lexbuf.start_pos (* Decodes the -(v + 2) encoding back to the original integer value. *) let __private__mem_value lexbuf i = -(lexbuf.__private__mem.(i) + 2) let __private__num_mem_cells lexbuf = Array.length lexbuf.__private__mem let lexeme_start lexbuf = lexbuf.start_pos + lexbuf.offset let lexeme_bytes_start lexbuf = lexbuf.start_bytes_pos + lexbuf.bytes_offset let lexeme_end lexbuf = lexbuf.pos + lexbuf.offset let lexeme_bytes_end lexbuf = lexbuf.bytes_pos + lexbuf.bytes_offset let loc lexbuf = (lexbuf.start_pos + lexbuf.offset, lexbuf.pos + lexbuf.offset) let bytes_loc lexbuf = ( lexbuf.start_bytes_pos + lexbuf.bytes_offset, lexbuf.bytes_pos + lexbuf.bytes_offset ) let lexeme_length lexbuf = lexbuf.pos - lexbuf.start_pos let lexeme_bytes_length lexbuf = lexbuf.bytes_pos - lexbuf.start_bytes_pos (* The index in [buf] of the code point at [pos] in the lexeme, once checked that [len] code points from there are in the lexeme. [buf] is the buffer of [lexbuf], read once by the caller: the range is also checked against that very array, so that the caller can read it without bounds checks whatever happens to [lexbuf] in between. *) let sub_lexeme_offset name lexbuf buf pos len = let off = lexbuf.start_pos + pos in if pos < 0 || len < 0 || pos > lexbuf.pos - lexbuf.start_pos - len || off > Array.length buf - len then invalid_arg name; off let sub_lexeme lexbuf pos len = let buf = lexbuf.buf in let off = sub_lexeme_offset "Sedlexing.sub_lexeme" lexbuf buf pos len in Array.sub buf off len type submatch = { lexbuf : lexbuf; pos : int; len : int } let lexeme_of_submatch s = sub_lexeme s.lexbuf s.pos s.len let lexeme lexbuf = Array.sub lexbuf.buf lexbuf.start_pos (lexbuf.pos - lexbuf.start_pos) let lexeme_char lexbuf pos = lexbuf.buf.(lexbuf.start_pos + pos) let lexing_position_start lexbuf = { Lexing.pos_fname = lexbuf.filename; pos_lnum = lexbuf.start_line; pos_cnum = lexbuf.start_pos + lexbuf.offset; pos_bol = lexbuf.start_bol; } let lexing_position_curr lexbuf = { Lexing.pos_fname = lexbuf.filename; pos_lnum = lexbuf.curr_line; pos_cnum = lexbuf.pos + lexbuf.offset; pos_bol = lexbuf.curr_bol; } let lexing_positions lexbuf = let start_p = lexing_position_start lexbuf and curr_p = lexing_position_curr lexbuf in (start_p, curr_p) let lexing_bytes_position_start lexbuf = { Lexing.pos_fname = lexbuf.filename; pos_lnum = lexbuf.start_line; pos_cnum = lexbuf.start_bytes_pos + lexbuf.bytes_offset; pos_bol = lexbuf.start_bytes_bol; } let lexing_bytes_position_curr lexbuf = { Lexing.pos_fname = lexbuf.filename; pos_lnum = lexbuf.curr_line; pos_cnum = lexbuf.bytes_pos + lexbuf.bytes_offset; pos_bol = lexbuf.curr_bytes_bol; } let lexing_bytes_positions lexbuf = let start_p = lexing_bytes_position_start lexbuf and curr_p = lexing_bytes_position_curr lexbuf in (start_p, curr_p) let with_tokenizer lexer' lexbuf = let lexer () = let token = lexer' lexbuf in let start_p, curr_p = lexing_positions lexbuf in (token, start_p, curr_p) in lexer module Chan = struct exception Missing_input type t = { b : Bytes.t; ic : in_channel; mutable len : int; mutable pos : int; } let min_buffer_size = 64 let create ic len : t = let len = max len min_buffer_size in { b = Bytes.create len; ic; len = 0; pos = 0 } let available (t : t) = t.len - t.pos let rec ensure_bytes_available (t : t) ~can_refill n = if available t >= n then () else if can_refill then ( let len = t.len - t.pos in if len > 0 then Bytes.blit t.b t.pos t.b 0 len; let read = input t.ic t.b len (Bytes.length t.b - len) in t.len <- len + read; t.pos <- 0; if read = 0 then raise Missing_input else ensure_bytes_available t ~can_refill n) else raise Missing_input let ensure_bytes_available t ~can_refill n = (* [n] should not exceed the size of the buffer. Here we are conservative and make sure it doesn't exceed the mininum size for the buffer. *) if n <= 0 || n > min_buffer_size then invalid_arg "Sedlexing.Chan.ensure"; ensure_bytes_available t ~can_refill n let get (t : t) i = Bytes.get t.b (t.pos + i) let advance (t : t) n = if t.pos + n > t.len then invalid_arg "advance"; t.pos <- t.pos + n let raw_buf (t : t) = t.b let raw_pos (t : t) = t.pos end let make_from_channel ?bytes_per_char ic ~max_bytes_per_uchar ~min_bytes_per_uchar ~read_uchar = let t = Chan.create ic (chunk_size * max_bytes_per_uchar) in let malformed = ref false in let refill buf pos len = let rec loop i = if !malformed then raise MalFormed; if i = len then i else ( match (* we refill our bytes buffer only if we haven't refilled any uchar yet. *) let can_refill = i = 0 in Chan.ensure_bytes_available t ~can_refill min_bytes_per_uchar; read_uchar ~can_refill t with | c -> buf.(pos + i) <- c; loop (i + 1) | exception MalFormed when i <> 0 -> malformed := true; i | exception Chan.Missing_input -> if i = 0 && Chan.available t > 0 then raise MalFormed; i) in loop 0 in create ?bytes_per_char refill module Latin1 = struct let from_gen s = from_gen ~bytes_per_char:(fun _ -> 1) (Gen.map Uchar.of_char s) let from_string s = let len = String.length s in { (empty_lexbuf (fun _ -> 1)) with buf = Array.init len (fun i -> Uchar.of_char s.[i]); len; finished = true; } let from_channel ic = make_from_channel ic ~bytes_per_char:(fun _ -> 1) ~min_bytes_per_uchar:1 ~max_bytes_per_uchar:1 ~read_uchar:(fun ~can_refill:_ t -> let c = Chan.get t 0 in Chan.advance t 1; Uchar.of_char c) let[@inline] to_latin1 c = if Uchar.is_char c then Uchar.unsafe_to_char c else raise (InvalidCodepoint (Uchar.to_int c)) let lexeme_char lexbuf pos = to_latin1 (lexeme_char lexbuf pos) let sub_lexeme lexbuf pos len = let buf = lexbuf.buf in let off = sub_lexeme_offset "Sedlexing.Latin1.sub_lexeme" lexbuf buf pos len in let s = Bytes.create len in for i = 0 to len - 1 do Bytes.unsafe_set s i (to_latin1 (Array.unsafe_get buf (off + i))) done; Bytes.unsafe_to_string s let lexeme lexbuf = sub_lexeme lexbuf 0 (lexbuf.pos - lexbuf.start_pos) let of_submatch s = sub_lexeme s.lexbuf s.pos s.len end module Utf8 = struct module Helper = struct (* http://www.faqs.org/rfcs/rfc3629.html *) let width = function | '\000' .. '\127' -> 1 | '\192' .. '\223' -> 2 | '\224' .. '\239' -> 3 | '\240' .. '\247' -> 4 | _ -> raise MalFormed (* https://www.unicode.org/versions/corrigendum1.html *) (* U+0080..U+07FF — no surrogate check needed, below U+D800 *) let check_two n1 n2 = if n1 < 0xc2 || 0xdf < n1 then raise MalFormed; if n2 < 0x80 || 0xbf < n2 then raise MalFormed; if n2 lsr 6 != 0b10 then raise MalFormed; ((n1 land 0x1f) lsl 6) lor (n2 land 0x3f) let check_three n1 n2 n3 = if n1 = 0xe0 then ( if n2 < 0xa0 || 0xbf < n2 then raise MalFormed; if n3 < 0x80 || 0xbf < n3 then raise MalFormed) else ( if n1 < 0xe1 || 0xef < n1 then raise MalFormed; if n2 < 0x80 || 0xbf < n2 then raise MalFormed; if n3 < 0x80 || 0xbf < n3 then raise MalFormed); if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 then raise MalFormed; let p = ((n1 land 0x0f) lsl 12) lor ((n2 land 0x3f) lsl 6) lor (n3 land 0x3f) in (* Reject UTF-16 surrogates (U+D800..U+DFFF) *) if p >= 0xd800 && p <= 0xdfff then raise MalFormed; p (* U+10000..U+10FFFF — no surrogate check needed, above U+DFFF *) let check_four n1 n2 n3 n4 = if n1 = 0xf0 then ( if n2 < 0x90 || 0xbf < n2 then raise MalFormed; if n3 < 0x80 || 0xbf < n3 then raise MalFormed; if n4 < 0x80 || 0xbf < n4 then raise MalFormed) else if n1 = 0xf4 then ( if n2 < 0x80 || 0x8f < n2 then raise MalFormed; if n3 < 0x80 || 0xbf < n3 then raise MalFormed; if n4 < 0x80 || 0xbf < n4 then raise MalFormed) else ( if n1 < 0xf1 || 0xf3 < n1 then raise MalFormed; if n2 < 0x80 || 0xbf < n2 then raise MalFormed; if n3 < 0x80 || 0xbf < n3 then raise MalFormed; if n4 < 0x80 || 0xbf < n4 then raise MalFormed); if n2 lsr 6 != 0b10 || n3 lsr 6 != 0b10 || n4 lsr 6 != 0b10 then raise MalFormed; ((n1 land 0x07) lsl 18) lor ((n2 land 0x3f) lsl 12) lor ((n3 land 0x3f) lsl 6) lor (n4 land 0x3f) let next s i = let c1 = s.[i] in match width c1 with | 1 -> Char.code c1 | 2 -> let n1 = Char.code c1 in let n2 = Char.code s.[i + 1] in check_two n1 n2 | 3 -> let n1 = Char.code c1 in let n2 = Char.code s.[i + 1] in let n3 = Char.code s.[i + 2] in check_three n1 n2 n3 | 4 -> let n1 = Char.code c1 in let n2 = Char.code s.[i + 1] in let n3 = Char.code s.[i + 2] in let n4 = Char.code s.[i + 3] in check_four n1 n2 n3 n4 | _ -> assert false let gen_from_char_gen s = let next_or_fail () = match Gen.next s with None -> raise MalFormed | Some x -> Char.code x in fun () -> Gen.next s >>| fun c1 -> match width c1 with | 1 -> Uchar.of_char c1 | 2 -> let n1 = Char.code c1 in let n2 = next_or_fail () in Uchar.of_int (check_two n1 n2) | 3 -> let n1 = Char.code c1 in let n2 = next_or_fail () in let n3 = next_or_fail () in Uchar.of_int (check_three n1 n2 n3) | 4 -> let n1 = Char.code c1 in let n2 = next_or_fail () in let n3 = next_or_fail () in let n4 = next_or_fail () in Uchar.of_int (check_four n1 n2 n3 n4) | _ -> raise MalFormed end let from_channel ic = make_from_channel ic ~bytes_per_char:Uchar.utf_8_byte_length ~min_bytes_per_uchar:1 ~max_bytes_per_uchar:4 ~read_uchar:(fun ~can_refill t -> let w = Helper.width (Chan.get t 0) in Chan.ensure_bytes_available t ~can_refill w; let c = Helper.next (Bytes.unsafe_to_string (Chan.raw_buf t)) (Chan.raw_pos t) in Chan.advance t w; Uchar.of_int c) let from_gen s = from_gen ~bytes_per_char:Uchar.utf_8_byte_length (Helper.gen_from_char_gen s) let from_string s = from_gen (Gen.init ~limit:(String.length s) (fun i -> String.get s i)) (* A first pass computes the size of the result, which is then written in place: no intermediate buffer. *) let sub_lexeme lexbuf pos len = let buf = lexbuf.buf in let off = sub_lexeme_offset "Sedlexing.Utf8.sub_lexeme" lexbuf buf pos len in let size = ref 0 in for i = off to off + len - 1 do size := !size + Uchar.utf_8_byte_length (Array.unsafe_get buf i) done; let s = Bytes.create !size in if !size = len then (* ASCII only, the common case: a plain copy is about 15% faster on lexemes of a few dozen characters. *) for i = 0 to len - 1 do Bytes.unsafe_set s i (Uchar.unsafe_to_char (Array.unsafe_get buf (off + i))) done else begin let j = ref 0 in for i = off to off + len - 1 do j := !j + Bytes.set_utf_8_uchar s !j (Array.unsafe_get buf i) done; (* the buffer was modified since the size was computed *) if !j <> !size then invalid_arg "Sedlexing.Utf8.sub_lexeme" end; Bytes.unsafe_to_string s let lexeme lexbuf = sub_lexeme lexbuf 0 (lexbuf.pos - lexbuf.start_pos) let of_submatch s = sub_lexeme s.lexbuf s.pos s.len end module Utf16 = struct type byte_order = Little_endian | Big_endian module Helper = struct (* http://www.ietf.org/rfc/rfc2781.txt *) let number_of_pair bo c1 c2 = match bo with | Little_endian -> (c2 lsl 8) + c1 | Big_endian -> (c1 lsl 8) + c2 let get_bo bo c1 c2 = match !bo with | Some o -> o | None -> let o = match (c1, c2) with | 0xff, 0xfe -> Little_endian | _ -> Big_endian in bo := Some o; o let gen_from_char_gen opt_bo s = let next_or_fail () = match Gen.next s with None -> raise MalFormed | Some x -> Char.code x in let bo = ref opt_bo in fun () -> Gen.next s >>| fun c1 -> let n1 = Char.code c1 in let n2 = next_or_fail () in let o = get_bo bo n1 n2 in let w1 = number_of_pair o n1 n2 in if w1 = 0xfffe then raise (InvalidCodepoint w1); if w1 < 0xd800 || 0xdfff < w1 then Uchar.of_int w1 else if w1 <= 0xdbff then ( let n3 = next_or_fail () in let n4 = next_or_fail () in let w2 = number_of_pair o n3 n4 in if w2 < 0xdc00 || w2 > 0xdfff then raise MalFormed; let upper10 = (w1 land 0x3ff) lsl 10 and lower10 = w2 land 0x3ff in Uchar.of_int (0x10000 + upper10 + lower10)) else raise MalFormed end let from_channel ic opt_bo = let bo = ref opt_bo in make_from_channel ic ~bytes_per_char:Uchar.utf_16_byte_length ~min_bytes_per_uchar:2 ~max_bytes_per_uchar:4 ~read_uchar:(fun ~can_refill t -> let n1 = Char.code (Chan.get t 0) in let n2 = Char.code (Chan.get t 1) in let o = Helper.get_bo bo n1 n2 in let w1 = Helper.number_of_pair o n1 n2 in if w1 = 0xfffe then raise (InvalidCodepoint w1); if w1 < 0xd800 || 0xdfff < w1 then ( Chan.advance t 2; Uchar.of_int w1) else if w1 <= 0xdbff then ( Chan.ensure_bytes_available t ~can_refill 4; let n3 = Char.code (Chan.get t 2) in let n4 = Char.code (Chan.get t 3) in let w2 = Helper.number_of_pair o n3 n4 in if w2 < 0xdc00 || w2 > 0xdfff then raise MalFormed; let upper10 = (w1 land 0x3ff) lsl 10 and lower10 = w2 land 0x3ff in Chan.advance t 4; Uchar.of_int (0x10000 + upper10 + lower10)) else raise MalFormed) let from_gen s opt_bo = from_gen ~bytes_per_char:Uchar.utf_16_byte_length (Helper.gen_from_char_gen opt_bo s) let from_string s = from_gen (Gen.init ~limit:(String.length s) (fun i -> String.get s i)) (* As for UTF-8: the size first, then the result written in place. The encoding loop is written once for each byte order, so that the encoder is inlined in it. *) let sub_lexeme lb pos len bo bom = let buf = lb.buf in let off = sub_lexeme_offset "Sedlexing.Utf16.sub_lexeme" lb buf pos len in let size = ref (if bom then 2 else 0) in for i = off to off + len - 1 do size := !size + Uchar.utf_16_byte_length (Array.unsafe_get buf i) done; let s = Bytes.create !size in let written = match bo with | Big_endian -> let j = ref (if bom then Bytes.set_utf_16be_uchar s 0 Uchar.bom else 0) in for i = off to off + len - 1 do j := !j + Bytes.set_utf_16be_uchar s !j (Array.unsafe_get buf i) done; !j | Little_endian -> let j = ref (if bom then Bytes.set_utf_16le_uchar s 0 Uchar.bom else 0) in for i = off to off + len - 1 do j := !j + Bytes.set_utf_16le_uchar s !j (Array.unsafe_get buf i) done; !j in (* the buffer was modified since the size was computed *) if written <> !size then invalid_arg "Sedlexing.Utf16.sub_lexeme"; Bytes.unsafe_to_string s let lexeme lb bo bom = sub_lexeme lb 0 (lb.pos - lb.start_pos) bo bom let of_submatch s bo bom = sub_lexeme s.lexbuf s.pos s.len bo bom end sedlex-3.8.1/src/lib/sedlexing.mli000066400000000000000000000421141526113672700170450ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) (** Runtime support for lexers generated by [sedlex]. *) (** This module is roughly equivalent to the module Lexing from the OCaml standard library, except that its lexbuffers handle Unicode code points (OCaml type: {!Uchar.t} in the range [0..0x10ffff]) instead of bytes (OCaml type: [char]). It is possible to have sedlex-generated lexers work on a custom implementation for lex buffers. To do this, define a module [L] which implements the [start], [next], [mark] and [backtrack] functions (See the Internal Interface section below for a specification). They need not work on a type named [lexbuf]: you can use the type name you want. Then, just do in your sedlex-processed source, bind this module to the name [Sedlexing] (for instance, with a local module definition: [let module Sedlexing = L in ...]. Of course, you'll probably want to define functions like [lexeme] to be used in the lexers semantic actions. *) (** The type of lexer buffers. A lexer buffer is the argument passed to the scanning functions defined by the generated lexers. The lexer buffer holds the internal information for the scanners, including the code points of the token currently scanned, its position from the beginning of the input stream, and the current position of the lexer. *) type lexbuf (** Raised by some functions to signal that some code point is not compatible with a specified encoding. *) exception InvalidCodepoint of int (** Raised by functions in the [Utf8] and [Utf16] modules to report strings which do not comply to the encoding. *) exception MalFormed (** {6 Creating generic lexbufs} *) (** Create a generic lexer buffer. When the lexer needs more characters, it will call the given function, giving it an array of Uchars [a], a position [pos] and a code point count [n]. The function should put [n] code points or less in [a], starting at position [pos], and return the number of characters provided. A return value of 0 means end of input. [bytes_per_char] argument is optional. If unspecified, byte positions are the same as code point position. *) val create : ?bytes_per_char:(Uchar.t -> int) -> (Uchar.t array -> int -> int -> int) -> lexbuf (** Set the initial tracked input position, in code points, for [lexbuf]. If unspecified, byte position is set to the same value as code point position. *) val set_position : ?bytes_position:Lexing.position -> lexbuf -> Lexing.position -> unit (** [set_filename lexbuf file] sets the filename to [file] in [lexbuf]. It also sets the {!Lexing.pos_fname} field in returned {!Lexing.position} records. *) val set_filename : lexbuf -> string -> unit (** Create a lexbuf from a stream of Unicode code points. [bytes_per_char] is optional. If unspecified, byte positions are the same as code point positions. *) val from_gen : ?bytes_per_char:(Uchar.t -> int) -> Uchar.t Gen.t -> lexbuf (** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is optional. If unspecified, byte positions are the same as code point positions. *) val from_int_array : ?bytes_per_char:(Uchar.t -> int) -> int array -> lexbuf (** Create a lexbuf from an array of Unicode code points. [bytes_per_char] is optional. If unspecified, byte positions are the same as code point positions. *) val from_uchar_array : ?bytes_per_char:(Uchar.t -> int) -> Uchar.t array -> lexbuf (** {6 Interface for lexers semantic actions} *) (** The following functions can be called from the semantic actions of lexer definitions. They give access to the character string matched by the regular expression associated with the semantic action. *) (** [Sedlexing.lexeme_start lexbuf] returns the offset in the input stream of the first code point of the matched string. The first code point of the stream has offset 0. *) val lexeme_start : lexbuf -> int (** [Sedlexing.lexeme_bytes_start lexbuf] returns the offset in the input stream of the first byte of the matched string. The first byte of the stream has offset 0. *) val lexeme_bytes_start : lexbuf -> int (** [Sedlexing.lexeme_end lexbuf] returns the offset in the input stream of the code point following the last code point of the matched string. The first code point of the stream has offset 0. *) val lexeme_end : lexbuf -> int (** [Sedlexing.lexeme_bytes_end lexbuf] returns the offset in the input stream of the byte following the last byte of the matched string. The first byte of the stream has offset 0. *) val lexeme_bytes_end : lexbuf -> int (** [Sedlexing.loc lexbuf] returns the pair [(Sedlexing.lexeme_start lexbuf,Sedlexing.lexeme_end lexbuf)]. *) val loc : lexbuf -> int * int (** [Sedlexing.bytes_loc lexbuf] returns the pair [(Sedlexing.lexeme_bytes_start lexbuf,Sedlexing.lexeme_bytes_end lexbuf)]. *) val bytes_loc : lexbuf -> int * int (** [Sedlexing.lexeme_length lexbuf] returns the difference [(Sedlexing.lexeme_end lexbuf) - (Sedlexing.lexeme_start lexbuf)], that is, the length (in code points) of the matched string. *) val lexeme_length : lexbuf -> int (** [Sedlexing.lexeme_bytes_length lexbuf] returns the difference [(Sedlexing.lexeme_bytes_end lexbuf) - (Sedlexing.lexeme_bytes_start lexbuf)], that is, the length (in bytes) of the matched string. *) val lexeme_bytes_length : lexbuf -> int (** [Sedlexing.lexing_positions lexbuf] returns the start and end positions, in code points, of the current token, using a record of type [Lexing.position]. This is intended for consumption by parsers like those generated by [Menhir]. *) val lexing_positions : lexbuf -> Lexing.position * Lexing.position (** [Sedlexing.lexing_position_start lexbuf] returns the start position, in code points, of the current token. *) val lexing_position_start : lexbuf -> Lexing.position (** [Sedlexing.lexing_position_curr lexbuf] returns the end position, in code points, of the current token. *) val lexing_position_curr : lexbuf -> Lexing.position (** [Sedlexing.lexing_bytes_positions lexbuf] returns the start and end positions, in bytes, of the current token, using a record of type [Lexing.position]. This is intended for consumption by parsers like those generated by [Menhir]. *) val lexing_bytes_positions : lexbuf -> Lexing.position * Lexing.position (** [Sedlexing.lexing_bytes_position_start lexbuf] returns the start position, in bytes, of the current token. *) val lexing_bytes_position_start : lexbuf -> Lexing.position (** [Sedlexing.lexing_bytes_position_curr lexbuf] returns the end position, in bytes, of the current token. *) val lexing_bytes_position_curr : lexbuf -> Lexing.position (** [Sedlexing.new_line lexbuf] increments the line count and sets the beginning of line to the current position, as though a newline character had been encountered in the input. *) val new_line : lexbuf -> unit (** [Sedlexing.lexeme lexbuf] returns the string matched by the regular expression as an array of Unicode code points. *) val lexeme : lexbuf -> Uchar.t array (** [Sedlexing.lexeme_char lexbuf pos] returns code point number [pos] in the matched string. *) val lexeme_char : lexbuf -> int -> Uchar.t (** [Sedlexing.sub_lexeme lexbuf pos len] returns a substring of the string matched by the regular expression as an array of Unicode code points. @raise Invalid_argument if [pos] and [len] do not designate a range of the matched string. *) val sub_lexeme : lexbuf -> int -> int -> Uchar.t array (** A submatch captures a sub-pattern matched by an [as] binding. It carries the lexbuf and the position/length of the submatch (in code points, relative to the start of the current token). Use the extraction functions below to obtain the matched content in the desired encoding. A submatch is a view into the lexbuf, valid only until the lexbuf moves on. Extracting it later returns another token's code points or raises [Invalid_argument]. Extract inside the action and keep the result. *) type submatch = { lexbuf : lexbuf; pos : int; len : int } (** [Sedlexing.lexeme_of_submatch s] returns the submatch as an array of Unicode code points. *) val lexeme_of_submatch : submatch -> Uchar.t array (** [Sedlexing.rollback lexbuf] puts [lexbuf] back in its configuration before the last lexeme was matched. It is then possible to use another lexer to parse the same characters again. The other functions above in this section should not be used in the semantic action after a call to [Sedlexing.rollback]. *) val rollback : lexbuf -> unit (** [with_tokenizer tokenizer lexbuf] given a lexer and a lexbuf, returns a generator of tokens annotated with positions. This generator can be used with the Menhir parser generator's incremental API. *) val with_tokenizer : (lexbuf -> 'token) -> lexbuf -> unit -> 'token * Lexing.position * Lexing.position (** {6 Internal interface} *) (** These functions are used internally by the lexers. They could be used to write lexers by hand, or with a lexer generator different from [sedlex]. The lexer buffers have a unique internal slot that can store an integer. They also store a "backtrack" position. *) (** [start lexbuf] informs the lexer buffer that any code points until the current position can be discarded. The current position becomes the "start" position as returned by [Sedlexing.lexeme_start]. Moreover, the internal slot is set to [-1] and the backtrack position is set to the current position. *) val start : lexbuf -> unit (** [next lexbuf] extracts the next code point from the lexer buffer and increments the current position. If the input stream is exhausted, the function returns [None]. If a ['\n'] is encountered, the tracked line number is incremented. *) val next : lexbuf -> Uchar.t option (** [mark lexbuf i] stores the integer [i] in the internal slot. The backtrack position is set to the current position. *) val mark : lexbuf -> int -> unit (** [backtrack lexbuf] returns the value stored in the internal slot of the buffer, and performs backtracking (the current position is set to the value of the backtrack position). Memory cells are left alone: those a rule's action reads are only written just before [mark]. *) val backtrack : lexbuf -> int (** [__private__next_int lexbuf] extracts the next code point from the lexer buffer and increments the current position. If the input stream is exhausted, the function returns -1. If a ['\n'] is encountered, the tracked line number is incremented. This is a private API, it should not be used by code using this module's API and can be removed at any time. *) val __private__next_int : lexbuf -> int (** Tagged DFA memory cells for [as] bindings. The following functions manage an internal array of memory cells used to record sub-match positions during DFA execution. Cells store either positions (>= 0) or encoded integer values (<= -2). Positions are automatically adjusted when the internal buffer is compacted, and converted to token-relative offsets on read by {!__private__mem_pos}. The cells are not cleared between tokens: only those written during the current match are meaningful. This is a private API used by generated code and may change at any time. *) (** [__private__ensure_mem lexbuf n] ensures at least [n] memory cells are available. Called once at the start of each [match%sedlex] block that uses [as] bindings. *) val __private__ensure_mem : lexbuf -> int -> unit (** [__private__set_mem_pos lexbuf i] records the current position in cell [i], for later retrieval by {!__private__mem_pos}. Used by [Set_position] tag operations on DFA transitions. *) val __private__set_mem_pos : lexbuf -> int -> unit (** [__private__set_mem_prev_pos lexbuf i] records in cell [i] the position before the code point just read: the current position minus one. *) val __private__set_mem_prev_pos : lexbuf -> int -> unit (** [__private__set_mem_value lexbuf i v] stores integer [v] in cell [i], encoded as [-(v + 2)] so it is disjoint from positions. Used by [Set_value] tag operations for or-pattern discriminators. *) val __private__set_mem_value : lexbuf -> int -> int -> unit (** [__private__copy_mem lexbuf dst src] copies the contents of cell [src] into cell [dst], preserving the position/value encoding. Used by [Copy] tag operations on DFA transitions. *) val __private__copy_mem : lexbuf -> int -> int -> unit (** [__private__mem_get lexbuf i] returns the raw contents of cell [i]. Used by generated code to save a cell in a local variable when a parallel register move both reads and overwrites it. The returned value is opaque; only pass it to {!__private__mem_set}. *) val __private__mem_get : lexbuf -> int -> int (** [__private__mem_set lexbuf i v] stores [v], a value previously obtained from {!__private__mem_get}, into cell [i]. *) val __private__mem_set : lexbuf -> int -> int -> unit (** [__private__mem_pos lexbuf i] returns the position stored in cell [i], as an offset relative to the start of the current token. *) val __private__mem_pos : lexbuf -> int -> int (** [__private__mem_value lexbuf i] decodes and returns the integer value stored in cell [i] (reverses the [-(v + 2)] encoding). *) val __private__mem_value : lexbuf -> int -> int (** Returns the current number of allocated memory cells. *) val __private__num_mem_cells : lexbuf -> int (** {6 Support for common encodings} *) module Latin1 : sig (** Create a lexbuf from a Latin1 encoded stream (ie a stream of Unicode code points in the range [0..255]) *) val from_gen : char Gen.t -> lexbuf (** Create a lexbuf from a Latin1 encoded input channel. The client is responsible for closing the channel. *) val from_channel : in_channel -> lexbuf (** Create a lexbuf from a Latin1 encoded string. *) val from_string : string -> lexbuf (** As [Sedlexing.lexeme] with a result encoded in Latin1. This function throws an exception [InvalidCodepoint] if it is not possible to encode the result in Latin1. *) val lexeme : lexbuf -> string (** As [Sedlexing.sub_lexeme] with a result encoded in Latin1. This function throws an exception [InvalidCodepoint] if it is not possible to encode the result in Latin1. *) val sub_lexeme : lexbuf -> int -> int -> string (** As [Sedlexing.lexeme_char] with a result encoded in Latin1. This function throws an exception [InvalidCodepoint] if it is not possible to encode the result in Latin1. *) val lexeme_char : lexbuf -> int -> char (** [of_submatch s] extracts the submatch as a Latin1 encoded string. *) val of_submatch : submatch -> string end module Utf8 : sig (** Create a lexbuf from a UTF-8 encoded stream. *) val from_gen : char Gen.t -> lexbuf (** Create a lexbuf from a UTF-8 encoded input channel. *) val from_channel : in_channel -> lexbuf (** Create a lexbuf from a UTF-8 encoded string. *) val from_string : string -> lexbuf (** As [Sedlexing.lexeme] with a result encoded in UTF-8. *) val lexeme : lexbuf -> string (** As [Sedlexing.sub_lexeme] with a result encoded in UTF-8. *) val sub_lexeme : lexbuf -> int -> int -> string (** [of_submatch s] extracts the submatch as a UTF-8 encoded string. *) val of_submatch : submatch -> string module Helper : sig val width : char -> int val check_two : int -> int -> int val check_three : int -> int -> int -> int val check_four : int -> int -> int -> int -> int end end module Utf16 : sig type byte_order = Little_endian | Big_endian (** [Utf16.from_gen s opt_bo] creates a lexbuf from an UTF-16 encoded stream. If [opt_bo] matches with [None] the function expects a BOM (Byte Order Mark), and takes the byte order as [Utf16.Big_endian] if it cannot find one. When [opt_bo] matches with [Some bo], [bo] is taken as byte order. In this case a leading BOM is kept in the stream - the lexer has to ignore it and a `wrong' BOM ([0xfffe]) will raise Utf16.InvalidCodepoint. *) val from_gen : char Gen.t -> byte_order option -> lexbuf (** Works as [Utf16.from_gen] with an [in_channel]. *) val from_channel : in_channel -> byte_order option -> lexbuf (** Works as [Utf16.from_gen] with a [string]. *) val from_string : string -> byte_order option -> lexbuf (** [utf16_lexeme lb bo bom] as [Sedlexing.lexeme] with a result encoded in UTF-16 in byte_order [bo] and starting with a BOM if [bom = true]. *) val lexeme : lexbuf -> byte_order -> bool -> string (** [sub_lexeme lb pos len bo bom] as [Sedlexing.sub_lexeme] with a result encoded in UTF-16 with byte order [bo] and starting with a BOM if [bom=true] *) val sub_lexeme : lexbuf -> int -> int -> byte_order -> bool -> string (** [of_submatch s bo bom] extracts the submatch as a UTF-16 encoded string with byte order [bo] and starting with a BOM if [bom=true]. *) val of_submatch : submatch -> byte_order -> bool -> string end sedlex-3.8.1/src/syntax/000077500000000000000000000000001526113672700151365ustar00rootroot00000000000000sedlex-3.8.1/src/syntax/dune000066400000000000000000000005441526113672700160170ustar00rootroot00000000000000(library (name sedlex_ppx) (public_name sedlex.ppx) (kind ppx_rewriter) (libraries ppxlib sedlex sedlex.compiler) (ppx_runtime_libraries sedlex) (preprocess (pps ppxlib.metaquot))) (rule (targets unicode.ml) (mode promote) (deps (:gen ../generator/gen_unicode.exe) (glob_files ../generator/data/*.txt)) (action (run %{gen} %{targets}))) sedlex-3.8.1/src/syntax/iso.ml000066400000000000000000000121641526113672700162660ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) open Sedlex_compiler.Cset let tr8876_ident_char = let l = [ (* ASCII *) (0x0041, 0x005a); (0x0061, 0x007a); (* Latin *) (0x00c0, 0x00d6); (0x00d8, 0x00f6); (0x00f8, 0x01f5); (0x01fa, 0x0217); (0x0250, 0x02a8); (* Greek *) (0x0384, 0x0384); (0x0388, 0x038a); (0x038c, 0x038c); (0x038e, 0x03a1); (0x03a3, 0x03ce); (0x03d0, 0x03d6); (0x03da, 0x03da); (0x03dc, 0x03dc); (0x03de, 0x03de); (0x03e0, 0x03e0); (0x03e2, 0x03f3); (* Cyrillic *) (0x0401, 0x040d); (0x040f, 0x044f); (0x0451, 0x045c); (0x045e, 0x0481); (0x0490, 0x04c4); (0x04c7, 0x04c4); (0x04cb, 0x04cc); (0x04d0, 0x04eb); (0x04ee, 0x04f5); (0x04f8, 0x04f9); (* Armenian *) (0x0531, 0x0556); (0x0561, 0x0587); (0x04d0, 0x04eb); (* Hebrew *) (0x05d0, 0x05ea); (0x05f0, 0x05f4); (* Arabic *) (0x0621, 0x063a); (0x0640, 0x0652); (0x0670, 0x06b7); (0x06ba, 0x06be); (0x06c0, 0x06ce); (0x06e5, 0x06e7); (* Devanagari *) (0x0905, 0x0939); (0x0958, 0x0962); (* Bengali *) (0x0985, 0x098c); (0x098f, 0x0990); (0x0993, 0x09a8); (0x09aa, 0x09b0); (0x09b2, 0x09b2); (0x09b6, 0x09b9); (0x09dc, 0x09dd); (0x09df, 0x09e1); (0x09f0, 0x09f1); (* Gurmukhi *) (0x0a05, 0x0a0a); (0x0a0f, 0x0a10); (0x0a13, 0x0a28); (0x0a2a, 0x0a30); (0x0a32, 0x0a33); (0x0a35, 0x0a36); (0x0a38, 0x0a39); (0x0a59, 0x0a5c); (0x0a5e, 0x0a5e); (* Gunjarati *) (0x0a85, 0x0a8b); (0x0a8d, 0x0a8d); (0x0a8f, 0x0a91); (0x0a93, 0x0aa8); (0x0aaa, 0x0ab0); (0x0ab2, 0x0ab3); (0x0ab5, 0x0ab9); (0x0ae0, 0x0ae0); (* Oriya *) (0x0b05, 0x0b0c); (0x0b0f, 0x0b10); (0x0b13, 0x0b28); (0x0b2a, 0x0b30); (0x0b32, 0x0b33); (0x0b36, 0x0b39); (0x0b5c, 0x0b5d); (0x0b5f, 0x0b61); (* Tamil *) (0x0b85, 0x0b8a); (0x0b8e, 0x0b90); (0x0b92, 0x0b95); (0x0b99, 0x0b9a); (0x0b9c, 0x0b9c); (0x0b9e, 0x0b9f); (0x0ba3, 0x0ba4); (0x0ba8, 0x0baa); (0x0bae, 0x0bb5); (0x0bb7, 0x0bb9); (* Telugu *) (0x0c05, 0x0c0c); (0x0c0e, 0x0c10); (0x0c12, 0x0c28); (0x0c2a, 0x0c33); (0x0c35, 0x0c39); (0x0c60, 0x0c61); (* Kannada *) (0x0c85, 0x0c8c); (0x0c8e, 0x0c90); (0x0c92, 0x0ca8); (0x0caa, 0x0cb3); (0x0cb5, 0x0cb9); (0x0ce0, 0x0ce1); (* Malayam *) (0x0d05, 0x0d0c); (0x0d0e, 0x0d10); (0x0d12, 0x0d28); (0x0d2a, 0x0d39); (0x0d60, 0x0d61); (* Thai *) (0x0e01, 0x0e30); (0x0e32, 0x0e33); (0x0e40, 0x0e46); (0x0e4f, 0x0e5b); (* Lao *) (0x0e81, 0x0e82); (0x0e84, 0x0e84); (0x0e87, 0x0e88); (0x0e8a, 0x0e8a); (0x0e0d, 0x0e0d); (0x0e94, 0x0e97); (0x0e99, 0x0e9f); (0x0ea1, 0x0ea3); (0x0ea5, 0x0ea5); (0x0ea7, 0x0ea7); (0x0eaa, 0x0eab); (0x0ead, 0x0eb0); (0x0eb2, 0x0eb3); (0x0ebd, 0x0ebd); (0x0ec0, 0x0ec4); (0x0ec6, 0x0ec6); (* Georgian *) (0x10a0, 0x10c5); (0x10d0, 0x10f6); (* Hangul Jamo *) (0x1100, 0x1159); (0x1161, 0x11a2); (0x11a8, 0x11f9); (0x11d0, 0x11f6); (* Latin extensions *) (0x1e00, 0x1e9a); (0x1ea0, 0x1ef9); (* Greek extended *) (0x1f00, 0x1f15); (0x1f18, 0x1f1d); (0x1f20, 0x1f45); (0x1f48, 0x1f4d); (0x1f50, 0x1f57); (0x1f59, 0x1f59); (0x1f5b, 0x1f5b); (0x1f5d, 0x1f5d); (0x1f5f, 0x1f7d); (0x1f80, 0x1fb4); (0x1fb6, 0x1fbc); (0x1fc2, 0x1fc4); (0x1fc6, 0x1fcc); (0x1fd0, 0x1fd3); (0x1fd6, 0x1fdb); (0x1fe0, 0x1fec); (0x1ff2, 0x1ff4); (0x1ff6, 0x1ffc); (* Hiragana *) (0x3041, 0x3094); (0x309b, 0x309e); (* Katakana *) (0x30a1, 0x30fe); (* Bopmofo *) (0x3105, 0x312c); (* CJK Unified Ideographs *) (0x4e00, 0x9fa5); (* CJK Compatibility Ideographs *) (0xf900, 0xfa2d); (* Arabic Presentation Forms *) (0xfb1f, 0xfb36); (0xfb38, 0xfb3c); (0xfb3e, 0xfb3e); (0xfb40, 0xfb41); (0xfb42, 0xfb44); (0xfb46, 0xfbb1); (0xfbd3, 0xfd35); (* Arabic Presentation Forms-A *) (0xfd50, 0xfd85); (0xfd92, 0xfbc7); (0xfdf0, 0xfdfb); (* Arabic Presentation Forms-B *) (0xfe70, 0xfe72); (0xfe74, 0xfe74); (0xfe76, 0xfefc); (* Half width and Fullwidth Forms *) (0xff21, 0xff3a); (0xff41, 0xff5a); (0xff66, 0xffbe); (0xffc2, 0xffc7); (0xffca, 0xffcf); (0xffd2, 0xffd7); (0xffd2, 0xffd7); (0xffda, 0xffdc); ] in union_list (List.map (fun (a, b) -> interval a b) l) sedlex-3.8.1/src/syntax/iso.mli000066400000000000000000000004741526113672700164400ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) open Sedlex_compiler.Cset (** Letters to be used in identifiers, as specified by ISO .... *) (* Data provided by John M. Skaller *) val tr8876_ident_char : t sedlex-3.8.1/src/syntax/ppx_sedlex.ml000066400000000000000000001124421526113672700176470ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) open Ppxlib open Ast_builder.Default open Ast_helper open Sedlex_compiler (* let ocaml_version = Versions.ocaml_408 *) (* Decision tree for partitions. A partition maps Unicode code points to equivalence class indices. Rather than generating a flat lookup table (which would be huge), we build a binary decision tree that tests code points against split values. For dense regions below [limit], a compact byte-string table is used instead. [simplify] prunes unreachable branches given a known code-point range. *) let default_loc = Location.none type decision_tree = | Lte of int * decision_tree * decision_tree | Table of int * int array | Return of int let rec simplify_decision_tree (x : decision_tree) = match x with | Table _ | Return _ -> x | Lte (_, (Return a as l), Return b) when a = b -> l | Lte (i, l, r) -> ( let l = simplify_decision_tree l in let r = simplify_decision_tree r in match (l, r) with | Return a, Return b when a = b -> l | _ -> Lte (i, l, r)) (* [decision segments] builds a balanced binary decision tree from a sorted list of [(lo, hi, class_index)] segments. Pairs of adjacent segments are merged bottom-up into [Lte] nodes. Gaps between segments return -1 (no match). *) let decision l = let l = List.map (fun (a, b, i) -> (a, b, Return i)) l in let rec merge2 = function | (a1, b1, d1) :: (a2, b2, d2) :: rest -> let x = if b1 + 1 = a2 then d2 else Lte (a2 - 1, Return (-1), d2) in (a1, b2, Lte (b1, d1, x)) :: merge2 rest | rest -> rest in let rec aux = function | [(a, b, d)] -> Lte (a - 1, Return (-1), Lte (b, d, Return (-1))) | [] -> Return (-1) | l -> aux (merge2 l) in aux l (* Code points below [limit] with class index < 255 are eligible for compact byte-string table lookup instead of a decision tree. *) let limit = 8192 (* [decision_table segments] partitions segments into a table-eligible prefix (dense, low code points) and a tree-handled suffix. The prefix becomes a [Table] node for O(1) lookup; the suffix uses [decision]. *) let decision_table l = let rec aux m accu = function | ((a, b, i) as x) :: rem when b < limit && i < 255 -> aux (min a m) (x :: accu) rem | rem -> (m, accu, rem) in let min, table, rest = aux max_int [] l in match table with | [] -> decision l | [(min, max, i)] -> Lte (min - 1, Return (-1), Lte (max, Return i, decision rest)) | (_, max, _) :: _ -> let arr = Array.make (max - min + 1) 0 in let set (a, b, i) = for j = a to b do arr.(j - min) <- i + 1 done in List.iter set table; Lte (min - 1, Return (-1), Lte (max, Table (min, arr), decision rest)) let rec simplify min max = function | Lte (i, yes, no) -> if i >= max then simplify min max yes else if i < min then simplify min max no else Lte (i, simplify min i yes, simplify (i + 1) max no) | x -> x (* [segments_of_partition p] flattens a partition (array of char sets, one per equivalence class) into a sorted list of [(lo, hi, class_index)] segments suitable for [decision_table]. *) let segments_of_partition p = let seg = ref [] in Array.iteri (fun i c -> List.iter (fun (a, b) -> seg := (a, b, i) :: !seg) (c : Cset.t :> (int * int) list)) p; List.sort (fun (a1, _, _) (a2, _, _) -> compare a1 a2) !seg (* [decision_table partition] builds a complete decision tree for a partition: extracts segments, builds the hybrid table/tree, then simplifies by pruning branches outside the valid code-point range. *) let decision_table p = simplify (-1) Cset.max_code (decision_table (segments_of_partition p)) (* Helpers to build AST *) let appfun s l = let loc = default_loc in eapply ~loc (evar ~loc s) l let glb_value name def = let loc = default_loc in pstr_value ~loc Nonrecursive [value_binding ~loc ~pat:(pvar ~loc name) ~expr:def] (* Named regexps *) module StringMap = Map.Make (struct type t = string let compare = compare end) let builtin_regexps = List.fold_left (fun acc (n, c) -> StringMap.add n (Ir.chars c) acc) StringMap.empty ([ ("any", Cset.any); ("eof", Cset.eof); ("xml_letter", Xml.letter); ("xml_digit", Xml.digit); ("xml_extender", Xml.extender); ("xml_base_char", Xml.base_char); ("xml_ideographic", Xml.ideographic); ("xml_combining_char", Xml.combining_char); ("xml_blank", Xml.blank); ("tr8876_ident_char", Iso.tr8876_ident_char); ] @ Unicode.Categories.list @ Unicode.Properties.list) (* Tables (indexed mapping: codepoint -> next state) *) let tables = Hashtbl.create 31 let get_tables () = Hashtbl.fold (fun key x accu -> (x, key) :: accu) tables [] let table_name x = try Hashtbl.find tables x with Not_found -> let s = Printf.sprintf "__sedlex_table_%i" (Hashtbl.length tables + 1) in Hashtbl.add tables x s; s (* [table (name, v)] generates a top-level [let __sedlex_table_N = "..."] binding where the string encodes the byte array [v] (one byte per entry, used for compact partition lookup). *) let table (name, v) = let n = Array.length v in let s = Bytes.create n in for i = 0 to n - 1 do Bytes.set s i (Char.chr v.(i)) done; glb_value name (estring ~loc:default_loc (Bytes.to_string s)) (* Partition (function: codepoint -> next state) *) let partitions = Hashtbl.create 31 let get_partitions () = Hashtbl.fold (fun key x accu -> (x, key) :: accu) partitions [] let partition_name x = try Hashtbl.find partitions x with Not_found -> let s = Printf.sprintf "__sedlex_partition_%i" (Hashtbl.length partitions + 1) in Hashtbl.add partitions x s; s let reset_state () = Hashtbl.clear tables; Hashtbl.clear partitions (* [partition (name, p)] generates a top-level [let __sedlex_partition_N c = ...] function that maps a code point [c] to its equivalence class index using the decision tree built from partition [p]. The EOF case (-1) is handled naturally by the decision tree since [simplify] uses -1 as the lower bound. *) let partition (name, p) = let loc = default_loc in let rec gen_tree = function | Lte (i, yes, no) -> [%expr if c <= [%e eint ~loc i] then [%e gen_tree yes] else [%e gen_tree no]] | Return i -> eint ~loc:default_loc i | Table (offset, t) -> let c = if offset = 0 then [%expr c] else [%expr c - [%e eint ~loc offset]] in [%expr Char.code (String.unsafe_get [%e evar ~loc (table_name t)] [%e c]) - 1] in let body = gen_tree (simplify_decision_tree (decision_table p)) in glb_value name [%expr fun c -> let open! Stdlib in [%e body]] (* Code generation for the automata *) let state_fun state = Printf.sprintf "__sedlex_state_%i" state (* What [Set_position] records in the generated code. Transition operations execute before their character is consumed, but the code runs them after reading it. *) type position = | Current (** final operations, and end-of-input transitions: reading end of input does not advance *) | Previous (** character transitions: one code point back *) (* [gen_tag_ops ~position lexbuf ops cont] wraps [cont] in the code performing the tag operations [ops], which form a parallel move: every [Copy] must read its source as it was before any operation of the list executed. Sources that the list also writes are first saved in let-bound locals; everything else compiles to direct [__private__set_mem_pos] / [__private__set_mem_value] / [__private__copy_mem] calls in list order. *) let gen_tag_ops ~position lexbuf (ops : Sedlex.tag_op list) cont = let loc = default_loc in let dests = List.map Sedlex.op_dest ops in let clobbered = List.sort_uniq compare (List.filter_map (fun (op : Sedlex.tag_op) -> match op with | Copy { src; _ } when List.mem src dests -> Some src | _ -> None) ops) in let local s = Printf.sprintf "__sedlex_mem_%d" s in let writes = List.fold_right (fun (op : Sedlex.tag_op) acc -> match op with | Set_position { dst } -> ( let dst = eint ~loc dst in match position with | Current -> [%expr Sedlexing.__private__set_mem_pos [%e lexbuf] [%e dst]; [%e acc]] | Previous -> [%expr Sedlexing.__private__set_mem_prev_pos [%e lexbuf] [%e dst]; [%e acc]]) | Set_value { dst; value } -> [%expr Sedlexing.__private__set_mem_value [%e lexbuf] [%e eint ~loc dst] [%e eint ~loc value]; [%e acc]] | Copy { dst; src } when List.mem src clobbered -> [%expr Sedlexing.__private__mem_set [%e lexbuf] [%e eint ~loc dst] [%e evar ~loc (local src)]; [%e acc]] | Copy { dst; src } -> [%expr Sedlexing.__private__copy_mem [%e lexbuf] [%e eint ~loc dst] [%e eint ~loc src]; [%e acc]]) ops cont in List.fold_right (fun s acc -> [%expr let [%p pvar ~loc (local s)] = Sedlexing.__private__mem_get [%e lexbuf] [%e eint ~loc s] in [%e acc]]) clobbered writes (* [call_state lexbuf auto state] generates the expression that transitions into DFA [state]. If the state has no outgoing transitions (a sink), it executes the state's final tag operations and returns the accepting rule index directly, or backtracks if the state accepts nothing; otherwise it emits a function call to the generated state function. *) let call_state lexbuf (auto : Sedlex.dfa) state = let loc = default_loc in let { Sedlex.trans; accept } = auto.(state) in if Array.length trans = 0 then ( match accept with | Some { Sedlex.rule; final_ops } -> gen_tag_ops ~position:Current lexbuf final_ops (eint ~loc rule) | None -> (* Nothing can match from here, e.g. after [eof] in [eof, eof]. *) [%expr Sedlexing.backtrack [%e lexbuf]]) else appfun (state_fun state) [lexbuf] (* [gen_state (lexbuf_name, lexbuf) auto i {trans; accept}] generates the function [__sedlex_state_N] for DFA state [i]. The function: 1. If the state is accepting ([accept = Some { rule; final_ops }]), executes [final_ops] then calls [mark] to save the current position. The cells read by the rule's action are only written by [final_ops]. 2. Reads the next code point, maps it through the partition function to get an equivalence class index, then pattern-matches on that index. 3. Each transition arm executes its tag operations then calls the target state function (or returns the rule index for sink states). 4. The default arm calls [backtrack] to return the last accepted rule. Returns [] for states with no outgoing transitions (sinks), which [call_state] inlines. *) let gen_state (lexbuf_name, lexbuf) (auto : Sedlex.dfa) i { Sedlex.trans; accept } = let loc = default_loc in let partition = Array.map (fun (cs, _, _) -> cs) trans in let cases = Array.mapi (fun i (cs, j, tags) -> let position = if Cset.mem (-1) cs then ( (* End of input never shares a transition with characters. *) assert (Cset.is_empty (Cset.difference cs Cset.eof)); Current) else Previous in let rhs = gen_tag_ops ~position lexbuf tags (call_state lexbuf auto j) in case ~lhs:(pint ~loc i) ~guard:None ~rhs) trans in let cases = Array.to_list cases in let body () = pexp_match ~loc (appfun (partition_name partition) [[%expr Sedlexing.__private__next_int [%e lexbuf]]]) (cases @ [ case ~lhs:[%pat? _] ~guard:None ~rhs:[%expr Sedlexing.backtrack [%e lexbuf]]; ]) in let ret body = let lhs = pvar ~loc:lexbuf.pexp_loc lexbuf_name in [ value_binding ~loc ~pat:(pvar ~loc (state_fun i)) ~expr:(Exp.fun_ ~loc Nolabel None lhs body); ] in match accept with | _ when Array.length trans = 0 -> [] | None -> ret (body ()) | Some { Sedlex.rule; final_ops } -> ret (gen_tag_ops ~position:Current lexbuf final_ops [%expr Sedlexing.mark [%e lexbuf] [%e eint ~loc rule]; [%e body ()]]) (* [gen_recflag auto] determines whether the generated state functions need [let rec]. If every transition leads to a sink state (no further transitions), the functions are non-recursive; otherwise they are mutually recursive. *) let gen_recflag (auto : Sedlex.dfa) = (* The generated function is not recursive if the transitions end in states with no further transitions. *) try Array.iter (fun { Sedlex.trans; _ } -> Array.iter (fun (_, j, _) -> if Array.length auto.(j).Sedlex.trans > 0 then raise Exit) trans) auto; Nonrecursive with Exit -> Recursive (* [gen_definition lexbuf_with_name compiled cases error] generates the complete lexer expression for one [match%sedlex] block: - Defines all [__sedlex_state_N] functions via [let rec ... in]. - Emits a start sequence: [start lexbuf], then (if the pattern has [as] bindings) [ensure_mem], then calls state 0. - Wraps the result in a [match] on the returned rule index, dispatching to user-provided right-hand-side expressions, with [error] as default. *) let gen_definition ((_, lexbuf) as lexbuf_with_name) (compiled : Sedlex.compiled) l error = let loc = default_loc in let auto = compiled.dfa in let cases = List.mapi (fun i (_, e) -> case ~lhs:(pint ~loc i) ~guard:None ~rhs:e) l in let states = Array.mapi (gen_state lexbuf_with_name auto) auto in let states = List.flatten (Array.to_list states) in let start_expr = if compiled.num_tags > 0 then ( let ensure_mem = [%expr Sedlexing.__private__ensure_mem [%e lexbuf] [%e eint ~loc compiled.num_tags]] in pexp_sequence ~loc [%expr Sedlexing.start [%e lexbuf]] (pexp_sequence ~loc ensure_mem (call_state lexbuf auto 0))) else pexp_sequence ~loc [%expr Sedlexing.start [%e lexbuf]] (call_state lexbuf auto 0) in let match_expr = pexp_match ~loc start_expr (cases @ [case ~lhs:(ppat_any ~loc) ~guard:None ~rhs:error]) in (* [states] is empty when state 0 is an accepting sink (e.g. the pattern [""]): [call_state] inlines its result above, so there are no state functions to bind and [pexp_let] with no bindings would be invalid. *) match states with | [] -> match_expr | _ -> pexp_let ~loc (gen_recflag auto) states match_expr (* Lexer specification parser *) let codepoint i = if i < 0 || i > Cset.max_code then failwith (Printf.sprintf "Invalid Unicode code point: %i" i); i let char c = Cset.singleton (Char.code c) let uchar c = Cset.singleton (Uchar.to_int c) let err loc fmt = Printf.ksprintf (fun s -> raise (Location.Error (Location.Error.createf ~loc "Sedlex: %s" s))) fmt type encoding = Utf8 | Latin1 | Ascii let string_of_encoding = function | Utf8 -> "UTF-8" | Latin1 -> "Latin-1" | Ascii -> "ASCII" (* [rev_csets_of_string ~loc ~encoding s] decodes string [s] under [encoding] and returns a list of singleton char sets in reverse order (one per character). Used to build sequence regexps from string literals. *) let rev_csets_of_string ~loc ~encoding s = match encoding with | Utf8 -> Utf8.fold ~f:(fun acc _ x -> match x with | `Malformed _ -> err loc "Malformed %s string" (string_of_encoding encoding) | `Uchar c -> uchar c :: acc) [] s | Latin1 -> let l = ref [] in for i = 0 to String.length s - 1 do l := char s.[i] :: !l done; !l | Ascii -> let l = ref [] in for i = 0 to String.length s - 1 do match s.[i] with | '\x00' .. '\x7F' as c -> l := char c :: !l | _ -> err loc "Malformed %s string" (string_of_encoding encoding) done; !l (* Code generation for `as` bindings. [gen_binding_code] turns the [compiled_binding list] (from the compiler) into [let] bindings that extract sub-matches from the lexbuf's memory cells (or computed offsets). For or-patterns with discriminators it emits a chain of if/else checks on the discriminator cell to select the correct positions. *) (* [gen_pos_expr lexbuf pe] generates code that evaluates a [pos_expr] to an integer position (offset from token start, in code points). *) let gen_pos_expr lexbuf (pe : Sedlex.pos_expr) = let loc = default_loc in match pe with | Tag { tag; offset = 0 } -> [%expr Sedlexing.__private__mem_pos [%e lexbuf] [%e eint ~loc tag]] | Tag { tag; offset } -> [%expr Sedlexing.__private__mem_pos [%e lexbuf] [%e eint ~loc tag] + [%e eint ~loc offset]] | Start_plus n -> eint ~loc n | End_minus 0 -> [%expr Sedlexing.lexeme_length [%e lexbuf]] | End_minus n -> [%expr Sedlexing.lexeme_length [%e lexbuf] - [%e eint ~loc n]] (* [gen_sub_lexeme lexbuf st et] generates an expression that evaluates position expressions [st] and [et] and constructs a [Sedlexing.submatch] record. *) let gen_sub_lexeme lexbuf st et = let loc = default_loc in [%expr let __s = [%e gen_pos_expr lexbuf st] in let __e = [%e gen_pos_expr lexbuf et] in { Sedlexing.lexbuf = [%e lexbuf]; pos = __s; len = __e - __s }] (* [gen_binding_code lexbuf bindings action] wraps [action] with [let] bindings that extract sub-match values from the lexbuf's memory cells. For a single binding (no or-pattern), emits a direct sub_lexeme call. For or-patterns with multiple tag pairs, emits a chain of [if disc_cell = value then ...] to select the correct positions at runtime. *) let gen_binding_code lexbuf (bindings : Sedlex.compiled_binding list) action = let loc = default_loc in ignore loc; if bindings = [] then action else ( (* Group bindings by variable name *) let by_name = let tbl = Hashtbl.create 8 in let order = ref [] in List.iter (fun ({ name; start_pos; end_pos; disc } : Sedlex.compiled_binding) -> if not (Hashtbl.mem tbl name) then order := name :: !order; let existing = try Hashtbl.find tbl name with Not_found -> [] in Hashtbl.replace tbl name (existing @ [(start_pos, end_pos, disc)])) bindings; List.rev_map (fun name -> (name, Hashtbl.find tbl name)) !order in List.fold_right (fun (name, entries) acc -> match entries with | [(st, et, _)] -> [%expr let [%p pvar ~loc name] = [%e gen_sub_lexeme lexbuf st et] in [%e acc]] | _ -> (* All the discriminators on the way to an alternative are tested, the outermost included: the cells are not cleared between tokens, so an inner one may hold the value of an earlier token, and only counts once the outer ones agree. *) let gen_disc_cond discs = let check (cell, value) = [%expr Sedlexing.__private__mem_value [%e lexbuf] [%e eint ~loc cell] = [%e eint ~loc value]] in match discs with | [] -> assert false | [d] -> check d | d :: ds -> List.fold_left (fun acc d -> [%expr [%e acc] && [%e check d]]) (check d) ds in let rec gen_checks = function | [(st, et, _)] -> gen_sub_lexeme lexbuf st et | (st, et, discs) :: rest -> if discs = [] then failwith "discriminator required for or-pattern with multiple \ bindings"; [%expr if [%e gen_disc_cond discs] then [%e gen_sub_lexeme lexbuf st et] else [%e gen_checks rest]] | [] -> assert false in [%expr let [%p pvar ~loc name] = [%e gen_checks entries] in [%e acc]]) by_name action) (* [ir_of_pattern env] returns a function [pattern -> Ir.t] that parses an OCaml pattern AST into an IR node. [env] maps names to previously defined IR patterns. All tag allocation, discriminator handling, and fixed-length optimization are deferred to the compiler's [compile_ir]. *) let ir_of_pattern env = let unwrap loc = function Ok ir -> ir | Error msg -> err loc "%s" msg in (* An empty character set can never match; it would also leave the DFA with a dead state the code generator cannot express. Reject it where it is formed ([Chars ""], [Compl any], [Sub]/[Intersect] of coinciding classes) so the mistake is reported at its source. *) let nonempty loc r = match r with | Ir.Chars c when Cset.is_empty c -> err loc "empty character set: this pattern can never match" | r -> r in let rec char_pair_op func name ~encoding ~loc tuple = (* Construct something like Sub(a,b) *) match tuple with | Some { ppat_desc = Ppat_tuple [p0; p1]; _ } -> ( let r0 = unwrap p0.ppat_loc (Ir.reject_captures name (aux ~encoding p0)) in let r1 = unwrap p1.ppat_loc (Ir.reject_captures name (aux ~encoding p1)) in match func r0 r1 with | Some r -> nonempty loc r | None -> err loc "the %s operator can only applied to single-character length \ regexps" name) | _ -> err loc "the %s operator requires two arguments, like %s(a,b)" name name and ir_compl r = match r with | Ir.Chars c -> Some (Ir.Chars (Cset.difference Cset.any c)) | _ -> None and ir_subtract r0 r1 = match (r0, r1) with | Ir.Chars c0, Ir.Chars c1 -> Some (Ir.Chars (Cset.difference c0 c1)) | _ -> None and ir_intersection r0 r1 = match (r0, r1) with | Ir.Chars c0, Ir.Chars c1 -> Some (Ir.Chars (Cset.intersection c0 c1)) | _ -> None and aux ~encoding p = match p.ppat_desc with (* name as x — named sub-match binding *) | Ppat_alias (inner, { txt = name; loc = name_loc }) -> unwrap name_loc (Ir.capture name (aux ~encoding inner)) (* p1 | p2 — alternation *) | Ppat_or (p1, p2) -> unwrap p.ppat_loc (Ir.alt (aux ~encoding p1) (aux ~encoding p2)) (* (p1, p2, ...) — sequence *) | Ppat_tuple (p :: pl) -> List.fold_left (fun acc p -> unwrap p.ppat_loc (Ir.seq acc (aux ~encoding p))) (aux ~encoding p) pl (* Star p — zero-or-more repetition *) | Ppat_construct ({ txt = Lident "Star"; _ }, Some (_, p)) -> unwrap p.ppat_loc (Ir.star (aux ~encoding p)) (* Plus p — one-or-more repetition *) | Ppat_construct ({ txt = Lident "Plus"; _ }, Some (_, p)) -> unwrap p.ppat_loc (Ir.plus (aux ~encoding p)) (* Utf8 p — switch to UTF-8 encoding *) | Ppat_construct ({ txt = Lident "Utf8"; _ }, Some (_, p)) -> aux ~encoding:Utf8 p (* Latin1 p — switch to Latin-1 encoding *) | Ppat_construct ({ txt = Lident "Latin1"; _ }, Some (_, p)) -> aux ~encoding:Latin1 p (* Ascii p — switch to ASCII encoding *) | Ppat_construct ({ txt = Lident "Ascii"; _ }, Some (_, p)) -> aux ~encoding:Ascii p (* Rep (p, n..m) — bounded repetition *) | Ppat_construct ( { txt = Lident "Rep"; _ }, Some ( _, { ppat_desc = Ppat_tuple [ p0; { ppat_desc = Ppat_constant (i1 as i2) | Ppat_interval (i1, i2); _; }; ]; _; } ) ) -> ( let r = aux ~encoding p0 in match (i1, i2) with | Pconst_integer (i1, _), Pconst_integer (i2, _) -> let i1 = int_of_string i1 in let i2 = int_of_string i2 in (* Check bounds explicitly here for better error locations *) if 0 <= i1 && i1 <= i2 then unwrap p0.ppat_loc (Ir.rep r i1 i2) else err p.ppat_loc "Invalid range for Rep operator" | _ -> err p.ppat_loc "Rep must take an integer constant or interval") (* Rep _ — malformed *) | Ppat_construct ({ txt = Lident "Rep"; _ }, _) -> err p.ppat_loc "the Rep operator takes 2 arguments" (* Opt p — optional (zero or one). [alt r eps], consume first: greedy like Star/Plus/Rep, so [Opt p] and [Rep (p, 0 .. 1)] agree. *) | Ppat_construct ({ txt = Lident "Opt"; _ }, Some (_, p)) -> let r = unwrap p.ppat_loc (Ir.reject_captures "Opt" (aux ~encoding p)) in unwrap p.ppat_loc (Ir.alt r Ir.eps) (* Compl p — complement of a character class *) | Ppat_construct ({ txt = Lident "Compl"; _ }, arg) -> ( match arg with | Some (_, p0) -> ( let r = unwrap p0.ppat_loc (Ir.reject_captures "Compl" (aux ~encoding p0)) in match ir_compl r with | Some r -> nonempty p.ppat_loc r | None -> err p.ppat_loc "the Compl operator can only applied to a \ single-character length regexp") | _ -> err p.ppat_loc "the Compl operator requires an argument") (* Sub (a, b) — character class subtraction *) | Ppat_construct ({ txt = Lident "Sub"; _ }, arg) -> char_pair_op ~encoding ir_subtract "Sub" ~loc:p.ppat_loc (Option.map (fun (_, arg) -> arg) arg) (* Intersect (a, b) — character class intersection *) | Ppat_construct ({ txt = Lident "Intersect"; _ }, arg) -> char_pair_op ~encoding ir_intersection "Intersect" ~loc:p.ppat_loc (Option.map (fun (_, arg) -> arg) arg) (* Chars "..." — character set from string literal *) | Ppat_construct ({ txt = Lident "Chars"; _ }, arg) -> ( let const = match arg with | Some (_, { ppat_desc = Ppat_constant const; _ }) -> Some const | _ -> None in match const with | Some (Pconst_string (s, _, _)) -> let l = rev_csets_of_string ~loc:p.ppat_loc ~encoding s in let chars = List.fold_left Cset.union Cset.empty l in nonempty p.ppat_loc (Ir.chars chars) | _ -> err p.ppat_loc "the Chars operator requires a string argument") (* 'a' .. 'z' or 0x41 .. 0x5a — character/codepoint range *) | Ppat_interval (i_start, i_end) -> ( match (i_start, i_end) with | Pconst_char c1, Pconst_char c2 -> let valid = match encoding with (* utf8 char interval can only match ascii because of the OCaml lexer. *) | Ascii | Utf8 -> ( function '\x00' .. '\x7f' -> true | _ -> false) | Latin1 -> ( function _ -> true) in if not (valid c1 && valid c2) then err p.ppat_loc "this pattern is not a valid %s interval regexp" (string_of_encoding encoding); Ir.chars (Cset.interval (Char.code c1) (Char.code c2)) | Pconst_integer (i1, _), Pconst_integer (i2, _) -> Ir.chars (Cset.interval (codepoint (int_of_string i1)) (codepoint (int_of_string i2))) | _ -> err p.ppat_loc "this pattern is not a valid interval regexp") (* "string" or 'c' or 0x42 — literal string, char, or codepoint *) | Ppat_constant const -> ( match const with | Pconst_string (s, _, _) -> let rev_l = rev_csets_of_string s ~loc:p.ppat_loc ~encoding in List.fold_left (fun acc cset -> (* chars have no captures, so seq cannot fail *) Result.get_ok (Ir.seq (Ir.chars cset) acc)) Ir.eps rev_l | Pconst_char c -> Ir.chars (char c) | Pconst_integer (i, _) -> Ir.chars (Cset.singleton (codepoint (int_of_string i))) | _ -> err p.ppat_loc "this pattern is not a valid regexp") (* Bare operator without argument *) | Ppat_construct ( { txt = Lident (("Star" | "Plus" | "Opt" | "Utf8" | "Latin1" | "Ascii") as name); _; }, None ) -> err p.ppat_loc "the %s operator requires an argument" name (* Unknown constructor *) | Ppat_construct ({ txt = Lident name; _ }, _) -> err p.ppat_loc "unknown sedlex operator %s" name (* name — reference to a previously defined regexp *) | Ppat_var { txt = x; _ } -> ( match StringMap.find_opt x env with | Some ir -> ir | None -> err p.ppat_loc "unbound regexp %s" x) | _ -> err p.ppat_loc "this pattern is not a valid regexp" in aux ~encoding:Ascii (* [handle_sedlex_match_ ~env ~map_rhs match_expr] is the main entry point for compiling a [match%sedlex lexbuf with ...] expression. It: 1. Extracts the lexbuf identifier and match cases. 2. Parses each case's pattern into [Ir.t] via [ir_of_pattern]. 3. Compiles all patterns via [Sedlex.compile_ir] (tag allocation, DFA). 4. Applies [map_rhs] to each case's right-hand side. 5. Wraps each RHS with [gen_binding_code] for [as] binding extraction. 6. Generates the full lexer code via [gen_definition]. Returns [(generated_expr, dfa)] for use by the main mapper and tests. *) let handle_sedlex_match_ ~env ~map_rhs match_expr = let lexbuf = match match_expr with | { pexp_desc = Pexp_match (lexbuf, _); _ } -> ( match lexbuf with | { pexp_desc = Pexp_ident { txt = Lident txt; _ }; _ } -> (txt, lexbuf) | _ -> err lexbuf.pexp_loc "the matched expression must be a single identifier") | _ -> err match_expr.pexp_loc "the %%sedlex extension is only recognized on match expressions" in let cases = match match_expr with | { pexp_desc = Pexp_match (_, cases); _ } -> cases | _ -> assert false in let cases = List.rev cases in let error = match List.hd cases with | { pc_lhs = [%pat? _]; pc_rhs = e; pc_guard = None } -> map_rhs e | { pc_lhs = p; _ } -> err p.ppat_loc "the last branch must be a catch-all error case" in let cases = List.rev (List.tl cases) in let cases_with_ir = List.map (function | { pc_lhs = p; pc_rhs = e; pc_guard = None } -> let ir = ir_of_pattern env p in (ir, p.ppat_loc, e) | { pc_guard = Some e; _ } -> err e.pexp_loc "'when' guards are not supported") cases in let compiled = Sedlex.compile_ir (Array.of_list (List.map (fun (ir, _, _) -> ir) cases_with_ir)) in (* map_rhs is called after compile_ir so that nested match%sedlex blocks (which call compile_ir, resetting tags) cannot corrupt the outer tag counter. *) let cases = List.mapi (fun i (_, _, e) -> let action = gen_binding_code (snd lexbuf) (Array.get compiled.bindings i) (map_rhs e) in ((), action)) cases_with_ir in let compiled_basic : Sedlex.compiled = { dfa = compiled.dfa; num_tags = compiled.num_tags } in (gen_definition lexbuf compiled_basic cases error, compiled.dfa) let handle_sedlex_match match_expr = handle_sedlex_match_ ~env:builtin_regexps ~map_rhs:Fun.id match_expr let previous = ref [] let regexps = ref [] let should_set_cookies = ref false (* The ppxlib AST mapper. It carries an [env] of named regexps (built-in Unicode categories + user [let%sedlex.regexp] definitions). The [toplevel] flag distinguishes the outermost structure (where partition/table definitions are emitted) from nested modules. *) let mapper = object (this) inherit Ast_traverse.map as super val env = builtin_regexps method define_regexp name ir = {} method eval_regexp_expr e = match e with (* [%sedlex.regexp? ] *) | [%expr [%sedlex.regexp? [%p? p]]] -> let ir = ir_of_pattern env p in if not (Ir.SSet.is_empty (Ir.capture_names ir)) then err p.ppat_loc "'as' bindings are not allowed in regexp definitions"; Some ir (* let = [%sedlex.regexp? ] in *) | [%expr let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] = [%sedlex.regexp? [%p? p]] in [%e? body]] -> let ir = ir_of_pattern env p in if not (Ir.SSet.is_empty (Ir.capture_names ir)) then err p.ppat_loc "'as' bindings are not allowed in regexp definitions"; (this#define_regexp name ir)#eval_regexp_expr body | _ -> None method! expression e = match e with (* match%sedlex with ... *) | [%expr [%sedlex [%e? { pexp_desc = Pexp_match _; _ } as match_expr]]] -> fst (handle_sedlex_match_ ~env ~map_rhs:this#expression match_expr) (* let = in — intercept when is a regexp *) | [%expr let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] = [%e? rhs] in [%e? _body]] -> ( match this#eval_regexp_expr rhs with | Some r -> (this#define_regexp name r)#expression _body | None -> super#expression e) (* [%sedlex ] — error *) | [%expr [%sedlex [%e? _]]] -> err e.pexp_loc "the %%sedlex extension is only recognized on match expressions" | _ -> super#expression e val toplevel = true method structure_with_regexps l = let mapper = ref this in let regexps = ref [] in let l = List.concat (List.map (function (* let = — intercept top-level regexp definitions *) | [%stri let [%p? { ppat_desc = Ppat_var { txt = name; _ }; _ }] = [%e? e]] as i -> ( match !mapper#eval_regexp_expr e with | Some r -> regexps := i :: !regexps; mapper := !mapper#define_regexp name r; [] | None -> [!mapper#structure_item i]) | i -> [!mapper#structure_item i]) l) in (l, List.rev !regexps) method! structure l = if toplevel then ( let sub = {} in let l, regexps' = sub#structure_with_regexps (!previous @ l) in let parts = List.map partition (get_partitions ()) in let tables = List.map table (get_tables ()) in regexps := regexps'; should_set_cookies := true; tables @ parts @ l) else fst (this#structure_with_regexps l) end let map_expression expr = mapper#expression expr (* ppxlib cookie handlers: regexp definitions survive across compilation units by round-tripping through a ppxlib cookie named "sedlex.regexps". *) let pre_handler cookies = previous := match Driver.Cookies.get cookies "sedlex.regexps" Ast_pattern.__ with | Some { pexp_desc = Pexp_extension (_, PStr l); _ } -> l | Some _ -> assert false | None -> [] let post_handler cookies = if !should_set_cookies then ( let loc = default_loc in Driver.Cookies.set cookies "sedlex.regexps" (pexp_extension ~loc ({ loc; txt = "regexps" }, PStr !regexps))) (* We register via ~impl:mapper#structure rather than ~extensions so that partition and table definitions can be inserted at the top level. *) (* let extensions = [ Extension.declare "sedlex" Extension.Context.expression Ast_pattern.(single_expr_payload __) (fun ~loc:_ ~path:_ expr -> mapper#expression expr); ] *) let () = Driver.Cookies.add_handler pre_handler; Driver.Cookies.add_post_handler post_handler; Driver.register_transformation "sedlex" ~impl:mapper#structure sedlex-3.8.1/src/syntax/ppx_sedlex.mli000066400000000000000000000025401526113672700200150ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (* Copyright 2005, 2013 by Alain Frisch and LexiFi. *) open Sedlex_compiler (** PPX rewriter for sedlex. Transforms [match%sedlex lexbuf with ...] into DFA-based lexer code. Registration with ppxlib is done via side effects at module initialization. *) (** {2 Internals exposed for the test PPX} The values below are not part of the public API. They are exposed so that [ppx_sedlex_test] can invoke the code generator in isolation and inspect the generated AST and DFA. *) (** Clear all internal tables (partitions, lookup tables). Must be called between independent compilations in tests. *) val reset_state : unit -> unit (** [handle_sedlex_match match_expr] compiles a [match%sedlex lexbuf with ...] expression into generated DFA code using the built-in regexp environment. Returns the generated expression and the DFA automaton. *) val handle_sedlex_match : Ppxlib.Parsetree.expression -> Ppxlib.Parsetree.expression * Sedlex.dfa (** [map_expression expr] applies the sedlex mapper to [expr], processing any [[%sedlex]] or [[%sedlex.regexp?]] extensions it contains. *) val map_expression : Ppxlib.Parsetree.expression -> Ppxlib.Parsetree.expression sedlex-3.8.1/src/syntax/unicode.ml000066400000000000000000005242061526113672700171270ustar00rootroot00000000000000[@@@ocamlformat "disable"] (* This file was automatically generated, do not edit. *) (* Edit gen_unicode.ml.inc instead. *) open Sedlex_compiler let version = "18.0.0" module Categories = struct let cc = Cset.of_list [0x0, 0x1f; 0x7f, 0x9f] let cf = Cset.of_list [0xad, 0xad; 0x600, 0x605; 0x61c, 0x61c; 0x6dd, 0x6dd; 0x70f, 0x70f; 0x890, 0x891; 0x8e2, 0x8e2; 0x180e, 0x180e; 0x200b, 0x200f; 0x202a, 0x202e; 0x2060, 0x2064; 0x2066, 0x206f; 0xfeff, 0xfeff; 0xfff9, 0xfffb; 0x110bd, 0x110bd; 0x110cd, 0x110cd; 0x13430, 0x1343f; 0x1bca0, 0x1bca3; 0x1d173, 0x1d17a; 0xe0001, 0xe0001; 0xe0020, 0xe007f] let cn = Cset.of_list [0x378, 0x379; 0x380, 0x383; 0x38b, 0x38b; 0x38d, 0x38d; 0x3a2, 0x3a2; 0x530, 0x530; 0x557, 0x557; 0x590, 0x590; 0x5ca, 0x5cf; 0x5eb, 0x5ee; 0x5f5, 0x5ff; 0x70e, 0x70e; 0x74b, 0x74c; 0x7b2, 0x7bf; 0x7fb, 0x7fc; 0x82e, 0x82f; 0x83f, 0x83f; 0x85c, 0x85d; 0x85f, 0x85f; 0x86b, 0x86f; 0x892, 0x896; 0x984, 0x984; 0x98d, 0x98e; 0x991, 0x992; 0x9a9, 0x9a9; 0x9b1, 0x9b1; 0x9b3, 0x9b5; 0x9ba, 0x9bb; 0x9c5, 0x9c6; 0x9c9, 0x9ca; 0x9cf, 0x9d6; 0x9d8, 0x9db; 0x9de, 0x9de; 0x9e4, 0x9e5; 0x9ff, 0xa00; 0xa04, 0xa04; 0xa0b, 0xa0e; 0xa11, 0xa12; 0xa29, 0xa29; 0xa31, 0xa31; 0xa34, 0xa34; 0xa37, 0xa37; 0xa3a, 0xa3b; 0xa3d, 0xa3d; 0xa43, 0xa46; 0xa49, 0xa4a; 0xa4e, 0xa50; 0xa52, 0xa58; 0xa5d, 0xa5d; 0xa5f, 0xa65; 0xa77, 0xa80; 0xa84, 0xa84; 0xa8e, 0xa8e; 0xa92, 0xa92; 0xaa9, 0xaa9; 0xab1, 0xab1; 0xab4, 0xab4; 0xaba, 0xabb; 0xac6, 0xac6; 0xaca, 0xaca; 0xace, 0xacf; 0xad1, 0xadf; 0xae4, 0xae5; 0xaf2, 0xaf8; 0xb00, 0xb00; 0xb04, 0xb04; 0xb0d, 0xb0e; 0xb11, 0xb12; 0xb29, 0xb29; 0xb31, 0xb31; 0xb34, 0xb34; 0xb3a, 0xb3b; 0xb45, 0xb46; 0xb49, 0xb4a; 0xb4e, 0xb52; 0xb58, 0xb5b; 0xb5e, 0xb5e; 0xb64, 0xb65; 0xb78, 0xb81; 0xb84, 0xb84; 0xb8b, 0xb8d; 0xb91, 0xb91; 0xb96, 0xb98; 0xb9b, 0xb9b; 0xb9d, 0xb9d; 0xba0, 0xba2; 0xba5, 0xba7; 0xbab, 0xbad; 0xbba, 0xbbd; 0xbc3, 0xbc5; 0xbc9, 0xbc9; 0xbce, 0xbcf; 0xbd1, 0xbd6; 0xbd8, 0xbe5; 0xbfb, 0xbff; 0xc0d, 0xc0d; 0xc11, 0xc11; 0xc29, 0xc29; 0xc3a, 0xc3b; 0xc45, 0xc45; 0xc49, 0xc49; 0xc4e, 0xc54; 0xc57, 0xc57; 0xc5b, 0xc5b; 0xc5e, 0xc5f; 0xc64, 0xc65; 0xc70, 0xc76; 0xc8d, 0xc8d; 0xc91, 0xc91; 0xca9, 0xca9; 0xcb4, 0xcb4; 0xcba, 0xcbb; 0xcc5, 0xcc5; 0xcc9, 0xcc9; 0xcce, 0xcd4; 0xcd7, 0xcdb; 0xcdf, 0xcdf; 0xce4, 0xce5; 0xcf0, 0xcf0; 0xcf4, 0xcff; 0xd0d, 0xd0d; 0xd11, 0xd11; 0xd45, 0xd45; 0xd49, 0xd49; 0xd50, 0xd53; 0xd64, 0xd65; 0xd80, 0xd80; 0xd84, 0xd84; 0xd97, 0xd99; 0xdb2, 0xdb2; 0xdbc, 0xdbc; 0xdbe, 0xdbf; 0xdc7, 0xdc9; 0xdcb, 0xdce; 0xdd5, 0xdd5; 0xdd7, 0xdd7; 0xde0, 0xde5; 0xdf0, 0xdf1; 0xdf5, 0xe00; 0xe3b, 0xe3e; 0xe5c, 0xe80; 0xe83, 0xe83; 0xe85, 0xe85; 0xe8b, 0xe8b; 0xea4, 0xea4; 0xea6, 0xea6; 0xebe, 0xebf; 0xec5, 0xec5; 0xec7, 0xec7; 0xecf, 0xecf; 0xeda, 0xedb; 0xee0, 0xeff; 0xf48, 0xf48; 0xf6d, 0xf70; 0xf98, 0xf98; 0xfbd, 0xfbd; 0xfcd, 0xfcd; 0xfdb, 0xfff; 0x10c6, 0x10c6; 0x10c8, 0x10cc; 0x10ce, 0x10cf; 0x1249, 0x1249; 0x124e, 0x124f; 0x1257, 0x1257; 0x1259, 0x1259; 0x125e, 0x125f; 0x1289, 0x1289; 0x128e, 0x128f; 0x12b1, 0x12b1; 0x12b6, 0x12b7; 0x12bf, 0x12bf; 0x12c1, 0x12c1; 0x12c6, 0x12c7; 0x12d7, 0x12d7; 0x1311, 0x1311; 0x1316, 0x1317; 0x135b, 0x135c; 0x137d, 0x137f; 0x139a, 0x139f; 0x13f6, 0x13f7; 0x13fe, 0x13ff; 0x169d, 0x169f; 0x16f9, 0x16ff; 0x1716, 0x171e; 0x1737, 0x173f; 0x1754, 0x175f; 0x176d, 0x176d; 0x1771, 0x1771; 0x1774, 0x177f; 0x17de, 0x17df; 0x17ea, 0x17ef; 0x17fa, 0x17ff; 0x181a, 0x181f; 0x1879, 0x187f; 0x18ab, 0x18af; 0x18f6, 0x18ff; 0x191f, 0x191f; 0x192c, 0x192f; 0x193c, 0x193f; 0x1941, 0x1943; 0x196e, 0x196f; 0x1975, 0x197f; 0x19ac, 0x19af; 0x19ca, 0x19cf; 0x19db, 0x19dd; 0x1a1c, 0x1a1d; 0x1a5f, 0x1a5f; 0x1a7d, 0x1a7e; 0x1a8a, 0x1a8f; 0x1a9a, 0x1a9f; 0x1aae, 0x1aaf; 0x1af1, 0x1aff; 0x1b4d, 0x1b4d; 0x1bf4, 0x1bfb; 0x1c38, 0x1c3a; 0x1c4a, 0x1c4c; 0x1c8b, 0x1c8f; 0x1cbb, 0x1cbc; 0x1cc8, 0x1ccf; 0x1cfb, 0x1cff; 0x1f16, 0x1f17; 0x1f1e, 0x1f1f; 0x1f46, 0x1f47; 0x1f4e, 0x1f4f; 0x1f58, 0x1f58; 0x1f5a, 0x1f5a; 0x1f5c, 0x1f5c; 0x1f5e, 0x1f5e; 0x1f7e, 0x1f7f; 0x1fb5, 0x1fb5; 0x1fc5, 0x1fc5; 0x1fd4, 0x1fd5; 0x1fdc, 0x1fdc; 0x1ff0, 0x1ff1; 0x1ff5, 0x1ff5; 0x1fff, 0x1fff; 0x2065, 0x2065; 0x2072, 0x2073; 0x20c5, 0x20cf; 0x20f1, 0x20ff; 0x218c, 0x218f; 0x242a, 0x243f; 0x244b, 0x245f; 0x2b74, 0x2b75; 0x2cf4, 0x2cf8; 0x2d26, 0x2d26; 0x2d28, 0x2d2c; 0x2d2e, 0x2d2f; 0x2d68, 0x2d6e; 0x2d71, 0x2d7e; 0x2d97, 0x2d9f; 0x2da7, 0x2da7; 0x2daf, 0x2daf; 0x2db7, 0x2db7; 0x2dbf, 0x2dbf; 0x2dc7, 0x2dc7; 0x2dcf, 0x2dcf; 0x2dd7, 0x2dd7; 0x2ddf, 0x2ddf; 0x2e5e, 0x2e5f; 0x2e64, 0x2e7f; 0x2e9a, 0x2e9a; 0x2ef4, 0x2eff; 0x2fd6, 0x2fef; 0x3040, 0x3040; 0x3097, 0x3098; 0x3100, 0x3104; 0x3130, 0x3130; 0x318f, 0x318f; 0x31e6, 0x31ee; 0x321f, 0x321f; 0xa48d, 0xa48f; 0xa4c7, 0xa4cf; 0xa62c, 0xa63f; 0xa6f8, 0xa6ff; 0xa7de, 0xa7e1; 0xa7e3, 0xa7f0; 0xa82d, 0xa82f; 0xa83a, 0xa83f; 0xa878, 0xa87f; 0xa8c6, 0xa8cd; 0xa8da, 0xa8df; 0xa954, 0xa95e; 0xa97d, 0xa97f; 0xa9ce, 0xa9ce; 0xa9da, 0xa9dd; 0xa9ff, 0xa9ff; 0xaa37, 0xaa3f; 0xaa4e, 0xaa4f; 0xaa5a, 0xaa5b; 0xaac3, 0xaada; 0xaaf7, 0xab00; 0xab07, 0xab08; 0xab0f, 0xab10; 0xab17, 0xab1f; 0xab27, 0xab27; 0xab2f, 0xab2f; 0xab6e, 0xab6f; 0xabee, 0xabef; 0xabfa, 0xabff; 0xd7a4, 0xd7af; 0xd7c7, 0xd7ca; 0xd7fc, 0xd7ff; 0xfa6e, 0xfa6f; 0xfada, 0xfaff; 0xfb07, 0xfb12; 0xfb18, 0xfb1c; 0xfb37, 0xfb37; 0xfb3d, 0xfb3d; 0xfb3f, 0xfb3f; 0xfb42, 0xfb42; 0xfb45, 0xfb45; 0xfdd0, 0xfdef; 0xfe1a, 0xfe1f; 0xfe53, 0xfe53; 0xfe67, 0xfe67; 0xfe6c, 0xfe6f; 0xfe75, 0xfe75; 0xfefd, 0xfefe; 0xff00, 0xff00; 0xffbf, 0xffc1; 0xffc8, 0xffc9; 0xffd0, 0xffd1; 0xffd8, 0xffd9; 0xffdd, 0xffdf; 0xffe7, 0xffe7; 0xffef, 0xfff8; 0xfffe, 0xffff; 0x1000c, 0x1000c; 0x10027, 0x10027; 0x1003b, 0x1003b; 0x1003e, 0x1003e; 0x1004e, 0x1004f; 0x1005e, 0x1007f; 0x100fb, 0x100ff; 0x10103, 0x10106; 0x10134, 0x10136; 0x1018f, 0x1018f; 0x1019d, 0x1019f; 0x101a1, 0x101cf; 0x101fe, 0x1027f; 0x1029d, 0x1029f; 0x102d1, 0x102df; 0x102fc, 0x102ff; 0x10324, 0x1032c; 0x1034b, 0x1034f; 0x1037b, 0x1037f; 0x1039e, 0x1039e; 0x103c4, 0x103c7; 0x103d6, 0x103ff; 0x1049e, 0x1049f; 0x104aa, 0x104af; 0x104d4, 0x104d7; 0x104fc, 0x104ff; 0x10528, 0x1052f; 0x10564, 0x1056e; 0x1057b, 0x1057b; 0x1058b, 0x1058b; 0x10593, 0x10593; 0x10596, 0x10596; 0x105a2, 0x105a2; 0x105b2, 0x105b2; 0x105ba, 0x105ba; 0x105bd, 0x105bf; 0x105f4, 0x105ff; 0x10737, 0x1073f; 0x10756, 0x1075f; 0x10768, 0x1077f; 0x10786, 0x10786; 0x107b1, 0x107b1; 0x107c0, 0x107ff; 0x10806, 0x10807; 0x10809, 0x10809; 0x10836, 0x10836; 0x10839, 0x1083b; 0x1083d, 0x1083e; 0x10856, 0x10856; 0x1089f, 0x108a6; 0x108b0, 0x108df; 0x108f3, 0x108f3; 0x108f6, 0x108fa; 0x1091c, 0x1091e; 0x1093a, 0x1093e; 0x1095a, 0x1097f; 0x109b8, 0x109bb; 0x109d0, 0x109d1; 0x10a04, 0x10a04; 0x10a07, 0x10a0b; 0x10a14, 0x10a14; 0x10a18, 0x10a18; 0x10a36, 0x10a37; 0x10a3b, 0x10a3e; 0x10a49, 0x10a4f; 0x10a59, 0x10a5f; 0x10aa0, 0x10abf; 0x10ae7, 0x10aea; 0x10af7, 0x10aff; 0x10b36, 0x10b38; 0x10b56, 0x10b57; 0x10b73, 0x10b77; 0x10b92, 0x10b98; 0x10b9d, 0x10ba8; 0x10bb0, 0x10bff; 0x10c49, 0x10c7f; 0x10cb3, 0x10cbf; 0x10cf3, 0x10cf9; 0x10d28, 0x10d2f; 0x10d3a, 0x10d3f; 0x10d66, 0x10d68; 0x10d86, 0x10d8d; 0x10d90, 0x10e5f; 0x10e7f, 0x10e7f; 0x10eaa, 0x10eaa; 0x10eae, 0x10eaf; 0x10eb2, 0x10ec1; 0x10ec8, 0x10ec8; 0x10eef, 0x10eef; 0x10f28, 0x10f2f; 0x10f5a, 0x10f6f; 0x10f8a, 0x10faf; 0x10fcc, 0x10fdf; 0x10ff7, 0x10fff; 0x1104e, 0x11051; 0x11076, 0x1107e; 0x110c3, 0x110cc; 0x110ce, 0x110cf; 0x110e9, 0x110ef; 0x110fa, 0x110ff; 0x11135, 0x11135; 0x11148, 0x1114f; 0x11177, 0x1117f; 0x111e0, 0x111e0; 0x111f5, 0x111ff; 0x11212, 0x11212; 0x11242, 0x1127f; 0x11287, 0x11287; 0x11289, 0x11289; 0x1128e, 0x1128e; 0x1129e, 0x1129e; 0x112aa, 0x112af; 0x112eb, 0x112ef; 0x112fa, 0x112ff; 0x11304, 0x11304; 0x1130d, 0x1130e; 0x11311, 0x11312; 0x11329, 0x11329; 0x11331, 0x11331; 0x11334, 0x11334; 0x1133a, 0x1133a; 0x11345, 0x11346; 0x11349, 0x1134a; 0x1134e, 0x1134f; 0x11351, 0x11356; 0x11358, 0x1135c; 0x11364, 0x11365; 0x1136d, 0x1136f; 0x11375, 0x1137f; 0x1138a, 0x1138a; 0x1138c, 0x1138d; 0x1138f, 0x1138f; 0x113b6, 0x113b6; 0x113c1, 0x113c1; 0x113c3, 0x113c4; 0x113c6, 0x113c6; 0x113cb, 0x113cb; 0x113d6, 0x113d6; 0x113d9, 0x113e0; 0x113e3, 0x113ff; 0x1145c, 0x1145c; 0x11462, 0x1147f; 0x114c8, 0x114cf; 0x114da, 0x1157f; 0x115b6, 0x115b7; 0x115de, 0x115ff; 0x11645, 0x1164f; 0x1165a, 0x1165f; 0x1166d, 0x1167f; 0x116ba, 0x116bf; 0x116ca, 0x116cf; 0x116e4, 0x116ff; 0x1171b, 0x1171c; 0x1172c, 0x1172f; 0x11747, 0x117ff; 0x1183c, 0x1189f; 0x118f3, 0x118fe; 0x11907, 0x11908; 0x1190a, 0x1190b; 0x11914, 0x11914; 0x11917, 0x11917; 0x11936, 0x11936; 0x11939, 0x1193a; 0x11947, 0x1194f; 0x1195a, 0x1199f; 0x119a8, 0x119a9; 0x119d8, 0x119d9; 0x119e5, 0x119ff; 0x11a48, 0x11a4f; 0x11aa3, 0x11aaf; 0x11af9, 0x11aff; 0x11b0b, 0x11b5f; 0x11b68, 0x11bbf; 0x11be2, 0x11bef; 0x11bfa, 0x11bff; 0x11c09, 0x11c09; 0x11c37, 0x11c37; 0x11c46, 0x11c4f; 0x11c6d, 0x11c6f; 0x11c90, 0x11c91; 0x11ca8, 0x11ca8; 0x11cb7, 0x11cff; 0x11d07, 0x11d07; 0x11d0a, 0x11d0a; 0x11d37, 0x11d39; 0x11d3b, 0x11d3b; 0x11d3e, 0x11d3e; 0x11d48, 0x11d4f; 0x11d5a, 0x11d5f; 0x11d66, 0x11d66; 0x11d69, 0x11d69; 0x11d8f, 0x11d8f; 0x11d92, 0x11d92; 0x11d99, 0x11d9f; 0x11daa, 0x11daf; 0x11ddc, 0x11ddf; 0x11dea, 0x11def; 0x11df2, 0x11edf; 0x11ef9, 0x11eff; 0x11f11, 0x11f11; 0x11f3b, 0x11f3d; 0x11f5b, 0x11faf; 0x11fb1, 0x11fbf; 0x11ff2, 0x11ffe; 0x1239a, 0x123ff; 0x12544, 0x1254f; 0x12687, 0x12f8f; 0x12ff3, 0x12fff; 0x13456, 0x1345f; 0x143fb, 0x143ff; 0x14647, 0x160ff; 0x1613a, 0x167ff; 0x16a39, 0x16a3f; 0x16a5f, 0x16a5f; 0x16a6a, 0x16a6d; 0x16abf, 0x16abf; 0x16aca, 0x16acf; 0x16aee, 0x16aef; 0x16af6, 0x16aff; 0x16b46, 0x16b4f; 0x16b5a, 0x16b5a; 0x16b62, 0x16b62; 0x16b78, 0x16b7c; 0x16b90, 0x16d3f; 0x16d7a, 0x16e3f; 0x16e9b, 0x16e9f; 0x16eb9, 0x16eba; 0x16ed4, 0x16eff; 0x16f4b, 0x16f4e; 0x16f88, 0x16f8e; 0x16fa0, 0x16fdf; 0x16fe5, 0x16fef; 0x16ff7, 0x16fff; 0x18cdb, 0x18cfe; 0x18d21, 0x18d7f; 0x18df3, 0x18dff; 0x19192, 0x1919f; 0x191d3, 0x1afef; 0x1aff4, 0x1aff4; 0x1affc, 0x1affc; 0x1afff, 0x1afff; 0x1b129, 0x1b131; 0x1b133, 0x1b14f; 0x1b153, 0x1b154; 0x1b156, 0x1b163; 0x1b169, 0x1b16f; 0x1b2fc, 0x1bbff; 0x1bc6b, 0x1bc6f; 0x1bc7d, 0x1bc7f; 0x1bc89, 0x1bc8f; 0x1bc9a, 0x1bc9b; 0x1bca4, 0x1cbff; 0x1ccfd, 0x1ccff; 0x1ceb4, 0x1ceb9; 0x1ced1, 0x1ced1; 0x1ced5, 0x1cedc; 0x1cefe, 0x1ceff; 0x1cf2e, 0x1cf2f; 0x1cf47, 0x1cf4f; 0x1cfc4, 0x1cfff; 0x1d0f6, 0x1d0ff; 0x1d246, 0x1d24f; 0x1d282, 0x1d2bf; 0x1d2d4, 0x1d2df; 0x1d2f4, 0x1d2ff; 0x1d357, 0x1d35f; 0x1d379, 0x1d3ff; 0x1d455, 0x1d455; 0x1d49d, 0x1d49d; 0x1d4a0, 0x1d4a1; 0x1d4a3, 0x1d4a4; 0x1d4a7, 0x1d4a8; 0x1d4ad, 0x1d4ad; 0x1d4ba, 0x1d4ba; 0x1d4bc, 0x1d4bc; 0x1d4c4, 0x1d4c4; 0x1d506, 0x1d506; 0x1d50b, 0x1d50c; 0x1d515, 0x1d515; 0x1d51d, 0x1d51d; 0x1d53a, 0x1d53a; 0x1d53f, 0x1d53f; 0x1d545, 0x1d545; 0x1d547, 0x1d549; 0x1d551, 0x1d551; 0x1d6a7, 0x1d6a7; 0x1d7cc, 0x1d7cd; 0x1da8c, 0x1da9a; 0x1daa0, 0x1daa0; 0x1dab0, 0x1daff; 0x1db1d, 0x1deff; 0x1df82, 0x1df8f; 0x1df97, 0x1dfcc; 0x1e007, 0x1e007; 0x1e019, 0x1e01a; 0x1e022, 0x1e022; 0x1e025, 0x1e025; 0x1e02b, 0x1e02f; 0x1e06e, 0x1e08e; 0x1e090, 0x1e0ff; 0x1e12d, 0x1e12f; 0x1e13e, 0x1e13f; 0x1e14a, 0x1e14d; 0x1e150, 0x1e28f; 0x1e2af, 0x1e2bf; 0x1e2fa, 0x1e2fe; 0x1e300, 0x1e4cf; 0x1e4fa, 0x1e5cf; 0x1e5fb, 0x1e5fe; 0x1e600, 0x1e6bf; 0x1e6df, 0x1e6df; 0x1e6f6, 0x1e6fd; 0x1e700, 0x1e7df; 0x1e7e7, 0x1e7e7; 0x1e7ec, 0x1e7ec; 0x1e7ef, 0x1e7ef; 0x1e7ff, 0x1e7ff; 0x1e8c5, 0x1e8c6; 0x1e8d7, 0x1e8ff; 0x1e94c, 0x1e94f; 0x1e95a, 0x1e95d; 0x1e960, 0x1ec70; 0x1ecb5, 0x1ed00; 0x1ed3e, 0x1edff; 0x1ee04, 0x1ee04; 0x1ee20, 0x1ee20; 0x1ee23, 0x1ee23; 0x1ee25, 0x1ee26; 0x1ee28, 0x1ee28; 0x1ee33, 0x1ee33; 0x1ee38, 0x1ee38; 0x1ee3a, 0x1ee3a; 0x1ee3c, 0x1ee41; 0x1ee43, 0x1ee46; 0x1ee48, 0x1ee48; 0x1ee4a, 0x1ee4a; 0x1ee4c, 0x1ee4c; 0x1ee50, 0x1ee50; 0x1ee53, 0x1ee53; 0x1ee55, 0x1ee56; 0x1ee58, 0x1ee58; 0x1ee5a, 0x1ee5a; 0x1ee5c, 0x1ee5c; 0x1ee5e, 0x1ee5e; 0x1ee60, 0x1ee60; 0x1ee63, 0x1ee63; 0x1ee65, 0x1ee66; 0x1ee6b, 0x1ee6b; 0x1ee73, 0x1ee73; 0x1ee78, 0x1ee78; 0x1ee7d, 0x1ee7d; 0x1ee7f, 0x1ee7f; 0x1ee8a, 0x1ee8a; 0x1ee9c, 0x1eea0; 0x1eea4, 0x1eea4; 0x1eeaa, 0x1eeaa; 0x1eebc, 0x1eeef; 0x1eef2, 0x1efff; 0x1f02c, 0x1f02f; 0x1f094, 0x1f09f; 0x1f0af, 0x1f0b0; 0x1f0c0, 0x1f0c0; 0x1f0d0, 0x1f0d0; 0x1f0f6, 0x1f0ff; 0x1f1af, 0x1f1e5; 0x1f203, 0x1f20f; 0x1f23c, 0x1f23f; 0x1f249, 0x1f24f; 0x1f252, 0x1f25f; 0x1f266, 0x1f2ff; 0x1f6da, 0x1f6db; 0x1f6ed, 0x1f6ef; 0x1f6fd, 0x1f6ff; 0x1f7dc, 0x1f7df; 0x1f7ec, 0x1f7ef; 0x1f80c, 0x1f80f; 0x1f848, 0x1f84f; 0x1f85a, 0x1f85f; 0x1f888, 0x1f88f; 0x1f8ae, 0x1f8af; 0x1f8bc, 0x1f8bf; 0x1f8c2, 0x1f8cf; 0x1f8d9, 0x1f8ff; 0x1fa58, 0x1fa5f; 0x1fa6e, 0x1fa6f; 0x1fa7d, 0x1fa7f; 0x1fac7, 0x1fac7; 0x1fac9, 0x1facb; 0x1fade, 0x1fade; 0x1faec, 0x1faee; 0x1fafb, 0x1faff; 0x1fb93, 0x1fb93; 0x1fbfb, 0x1ffff; 0x2a6e0, 0x2a6ff; 0x2b81f, 0x2b81f; 0x2ceae, 0x2ceaf; 0x2ebe1, 0x2ebef; 0x2ee5e, 0x2f7ff; 0x2fa1e, 0x2ffff; 0x3134b, 0x3134f; 0x3347a, 0x3cfff; 0x3fc40, 0xe0000; 0xe0002, 0xe001f; 0xe0080, 0xe00ff; 0xe01f0, 0xeffff; 0xffffe, 0xfffff; 0x10fffe, 0x10ffff] let co = Cset.of_list [0xe000, 0xf8ff; 0xf0000, 0xffffd; 0x100000, 0x10fffd] let cs = Cset.of_list [0xd800, 0xdfff] let ll = Cset.of_list [0x61, 0x7a; 0xb5, 0xb5; 0xdf, 0xf6; 0xf8, 0xff; 0x101, 0x101; 0x103, 0x103; 0x105, 0x105; 0x107, 0x107; 0x109, 0x109; 0x10b, 0x10b; 0x10d, 0x10d; 0x10f, 0x10f; 0x111, 0x111; 0x113, 0x113; 0x115, 0x115; 0x117, 0x117; 0x119, 0x119; 0x11b, 0x11b; 0x11d, 0x11d; 0x11f, 0x11f; 0x121, 0x121; 0x123, 0x123; 0x125, 0x125; 0x127, 0x127; 0x129, 0x129; 0x12b, 0x12b; 0x12d, 0x12d; 0x12f, 0x12f; 0x131, 0x131; 0x133, 0x133; 0x135, 0x135; 0x137, 0x138; 0x13a, 0x13a; 0x13c, 0x13c; 0x13e, 0x13e; 0x140, 0x140; 0x142, 0x142; 0x144, 0x144; 0x146, 0x146; 0x148, 0x149; 0x14b, 0x14b; 0x14d, 0x14d; 0x14f, 0x14f; 0x151, 0x151; 0x153, 0x153; 0x155, 0x155; 0x157, 0x157; 0x159, 0x159; 0x15b, 0x15b; 0x15d, 0x15d; 0x15f, 0x15f; 0x161, 0x161; 0x163, 0x163; 0x165, 0x165; 0x167, 0x167; 0x169, 0x169; 0x16b, 0x16b; 0x16d, 0x16d; 0x16f, 0x16f; 0x171, 0x171; 0x173, 0x173; 0x175, 0x175; 0x177, 0x177; 0x17a, 0x17a; 0x17c, 0x17c; 0x17e, 0x180; 0x183, 0x183; 0x185, 0x185; 0x188, 0x188; 0x18c, 0x18d; 0x192, 0x192; 0x195, 0x195; 0x199, 0x19b; 0x19e, 0x19e; 0x1a1, 0x1a1; 0x1a3, 0x1a3; 0x1a5, 0x1a5; 0x1a8, 0x1a8; 0x1aa, 0x1ab; 0x1ad, 0x1ad; 0x1b0, 0x1b0; 0x1b4, 0x1b4; 0x1b6, 0x1b6; 0x1b9, 0x1ba; 0x1bd, 0x1bf; 0x1c6, 0x1c6; 0x1c9, 0x1c9; 0x1cc, 0x1cc; 0x1ce, 0x1ce; 0x1d0, 0x1d0; 0x1d2, 0x1d2; 0x1d4, 0x1d4; 0x1d6, 0x1d6; 0x1d8, 0x1d8; 0x1da, 0x1da; 0x1dc, 0x1dd; 0x1df, 0x1df; 0x1e1, 0x1e1; 0x1e3, 0x1e3; 0x1e5, 0x1e5; 0x1e7, 0x1e7; 0x1e9, 0x1e9; 0x1eb, 0x1eb; 0x1ed, 0x1ed; 0x1ef, 0x1f0; 0x1f3, 0x1f3; 0x1f5, 0x1f5; 0x1f9, 0x1f9; 0x1fb, 0x1fb; 0x1fd, 0x1fd; 0x1ff, 0x1ff; 0x201, 0x201; 0x203, 0x203; 0x205, 0x205; 0x207, 0x207; 0x209, 0x209; 0x20b, 0x20b; 0x20d, 0x20d; 0x20f, 0x20f; 0x211, 0x211; 0x213, 0x213; 0x215, 0x215; 0x217, 0x217; 0x219, 0x219; 0x21b, 0x21b; 0x21d, 0x21d; 0x21f, 0x21f; 0x221, 0x221; 0x223, 0x223; 0x225, 0x225; 0x227, 0x227; 0x229, 0x229; 0x22b, 0x22b; 0x22d, 0x22d; 0x22f, 0x22f; 0x231, 0x231; 0x233, 0x239; 0x23c, 0x23c; 0x23f, 0x240; 0x242, 0x242; 0x247, 0x247; 0x249, 0x249; 0x24b, 0x24b; 0x24d, 0x24d; 0x24f, 0x293; 0x296, 0x2af; 0x371, 0x371; 0x373, 0x373; 0x377, 0x377; 0x37b, 0x37d; 0x390, 0x390; 0x3ac, 0x3ce; 0x3d0, 0x3d1; 0x3d5, 0x3d7; 0x3d9, 0x3d9; 0x3db, 0x3db; 0x3dd, 0x3dd; 0x3df, 0x3df; 0x3e1, 0x3e1; 0x3e3, 0x3e3; 0x3e5, 0x3e5; 0x3e7, 0x3e7; 0x3e9, 0x3e9; 0x3eb, 0x3eb; 0x3ed, 0x3ed; 0x3ef, 0x3f3; 0x3f5, 0x3f5; 0x3f8, 0x3f8; 0x3fb, 0x3fc; 0x430, 0x45f; 0x461, 0x461; 0x463, 0x463; 0x465, 0x465; 0x467, 0x467; 0x469, 0x469; 0x46b, 0x46b; 0x46d, 0x46d; 0x46f, 0x46f; 0x471, 0x471; 0x473, 0x473; 0x475, 0x475; 0x477, 0x477; 0x479, 0x479; 0x47b, 0x47b; 0x47d, 0x47d; 0x47f, 0x47f; 0x481, 0x481; 0x48b, 0x48b; 0x48d, 0x48d; 0x48f, 0x48f; 0x491, 0x491; 0x493, 0x493; 0x495, 0x495; 0x497, 0x497; 0x499, 0x499; 0x49b, 0x49b; 0x49d, 0x49d; 0x49f, 0x49f; 0x4a1, 0x4a1; 0x4a3, 0x4a3; 0x4a5, 0x4a5; 0x4a7, 0x4a7; 0x4a9, 0x4a9; 0x4ab, 0x4ab; 0x4ad, 0x4ad; 0x4af, 0x4af; 0x4b1, 0x4b1; 0x4b3, 0x4b3; 0x4b5, 0x4b5; 0x4b7, 0x4b7; 0x4b9, 0x4b9; 0x4bb, 0x4bb; 0x4bd, 0x4bd; 0x4bf, 0x4bf; 0x4c2, 0x4c2; 0x4c4, 0x4c4; 0x4c6, 0x4c6; 0x4c8, 0x4c8; 0x4ca, 0x4ca; 0x4cc, 0x4cc; 0x4ce, 0x4cf; 0x4d1, 0x4d1; 0x4d3, 0x4d3; 0x4d5, 0x4d5; 0x4d7, 0x4d7; 0x4d9, 0x4d9; 0x4db, 0x4db; 0x4dd, 0x4dd; 0x4df, 0x4df; 0x4e1, 0x4e1; 0x4e3, 0x4e3; 0x4e5, 0x4e5; 0x4e7, 0x4e7; 0x4e9, 0x4e9; 0x4eb, 0x4eb; 0x4ed, 0x4ed; 0x4ef, 0x4ef; 0x4f1, 0x4f1; 0x4f3, 0x4f3; 0x4f5, 0x4f5; 0x4f7, 0x4f7; 0x4f9, 0x4f9; 0x4fb, 0x4fb; 0x4fd, 0x4fd; 0x4ff, 0x4ff; 0x501, 0x501; 0x503, 0x503; 0x505, 0x505; 0x507, 0x507; 0x509, 0x509; 0x50b, 0x50b; 0x50d, 0x50d; 0x50f, 0x50f; 0x511, 0x511; 0x513, 0x513; 0x515, 0x515; 0x517, 0x517; 0x519, 0x519; 0x51b, 0x51b; 0x51d, 0x51d; 0x51f, 0x51f; 0x521, 0x521; 0x523, 0x523; 0x525, 0x525; 0x527, 0x527; 0x529, 0x529; 0x52b, 0x52b; 0x52d, 0x52d; 0x52f, 0x52f; 0x560, 0x588; 0x10d0, 0x10fa; 0x10fd, 0x10ff; 0x13f8, 0x13fd; 0x1c80, 0x1c88; 0x1c8a, 0x1c8a; 0x1d00, 0x1d2b; 0x1d6b, 0x1d77; 0x1d79, 0x1d9a; 0x1e01, 0x1e01; 0x1e03, 0x1e03; 0x1e05, 0x1e05; 0x1e07, 0x1e07; 0x1e09, 0x1e09; 0x1e0b, 0x1e0b; 0x1e0d, 0x1e0d; 0x1e0f, 0x1e0f; 0x1e11, 0x1e11; 0x1e13, 0x1e13; 0x1e15, 0x1e15; 0x1e17, 0x1e17; 0x1e19, 0x1e19; 0x1e1b, 0x1e1b; 0x1e1d, 0x1e1d; 0x1e1f, 0x1e1f; 0x1e21, 0x1e21; 0x1e23, 0x1e23; 0x1e25, 0x1e25; 0x1e27, 0x1e27; 0x1e29, 0x1e29; 0x1e2b, 0x1e2b; 0x1e2d, 0x1e2d; 0x1e2f, 0x1e2f; 0x1e31, 0x1e31; 0x1e33, 0x1e33; 0x1e35, 0x1e35; 0x1e37, 0x1e37; 0x1e39, 0x1e39; 0x1e3b, 0x1e3b; 0x1e3d, 0x1e3d; 0x1e3f, 0x1e3f; 0x1e41, 0x1e41; 0x1e43, 0x1e43; 0x1e45, 0x1e45; 0x1e47, 0x1e47; 0x1e49, 0x1e49; 0x1e4b, 0x1e4b; 0x1e4d, 0x1e4d; 0x1e4f, 0x1e4f; 0x1e51, 0x1e51; 0x1e53, 0x1e53; 0x1e55, 0x1e55; 0x1e57, 0x1e57; 0x1e59, 0x1e59; 0x1e5b, 0x1e5b; 0x1e5d, 0x1e5d; 0x1e5f, 0x1e5f; 0x1e61, 0x1e61; 0x1e63, 0x1e63; 0x1e65, 0x1e65; 0x1e67, 0x1e67; 0x1e69, 0x1e69; 0x1e6b, 0x1e6b; 0x1e6d, 0x1e6d; 0x1e6f, 0x1e6f; 0x1e71, 0x1e71; 0x1e73, 0x1e73; 0x1e75, 0x1e75; 0x1e77, 0x1e77; 0x1e79, 0x1e79; 0x1e7b, 0x1e7b; 0x1e7d, 0x1e7d; 0x1e7f, 0x1e7f; 0x1e81, 0x1e81; 0x1e83, 0x1e83; 0x1e85, 0x1e85; 0x1e87, 0x1e87; 0x1e89, 0x1e89; 0x1e8b, 0x1e8b; 0x1e8d, 0x1e8d; 0x1e8f, 0x1e8f; 0x1e91, 0x1e91; 0x1e93, 0x1e93; 0x1e95, 0x1e9d; 0x1e9f, 0x1e9f; 0x1ea1, 0x1ea1; 0x1ea3, 0x1ea3; 0x1ea5, 0x1ea5; 0x1ea7, 0x1ea7; 0x1ea9, 0x1ea9; 0x1eab, 0x1eab; 0x1ead, 0x1ead; 0x1eaf, 0x1eaf; 0x1eb1, 0x1eb1; 0x1eb3, 0x1eb3; 0x1eb5, 0x1eb5; 0x1eb7, 0x1eb7; 0x1eb9, 0x1eb9; 0x1ebb, 0x1ebb; 0x1ebd, 0x1ebd; 0x1ebf, 0x1ebf; 0x1ec1, 0x1ec1; 0x1ec3, 0x1ec3; 0x1ec5, 0x1ec5; 0x1ec7, 0x1ec7; 0x1ec9, 0x1ec9; 0x1ecb, 0x1ecb; 0x1ecd, 0x1ecd; 0x1ecf, 0x1ecf; 0x1ed1, 0x1ed1; 0x1ed3, 0x1ed3; 0x1ed5, 0x1ed5; 0x1ed7, 0x1ed7; 0x1ed9, 0x1ed9; 0x1edb, 0x1edb; 0x1edd, 0x1edd; 0x1edf, 0x1edf; 0x1ee1, 0x1ee1; 0x1ee3, 0x1ee3; 0x1ee5, 0x1ee5; 0x1ee7, 0x1ee7; 0x1ee9, 0x1ee9; 0x1eeb, 0x1eeb; 0x1eed, 0x1eed; 0x1eef, 0x1eef; 0x1ef1, 0x1ef1; 0x1ef3, 0x1ef3; 0x1ef5, 0x1ef5; 0x1ef7, 0x1ef7; 0x1ef9, 0x1ef9; 0x1efb, 0x1efb; 0x1efd, 0x1efd; 0x1eff, 0x1f07; 0x1f10, 0x1f15; 0x1f20, 0x1f27; 0x1f30, 0x1f37; 0x1f40, 0x1f45; 0x1f50, 0x1f57; 0x1f60, 0x1f67; 0x1f70, 0x1f7d; 0x1f80, 0x1f87; 0x1f90, 0x1f97; 0x1fa0, 0x1fa7; 0x1fb0, 0x1fb4; 0x1fb6, 0x1fb7; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fc7; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fd7; 0x1fe0, 0x1fe7; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ff7; 0x210a, 0x210a; 0x210e, 0x210f; 0x2113, 0x2113; 0x212f, 0x212f; 0x2134, 0x2134; 0x2139, 0x2139; 0x213c, 0x213d; 0x2146, 0x2149; 0x214e, 0x214e; 0x2184, 0x2184; 0x2c30, 0x2c5f; 0x2c61, 0x2c61; 0x2c65, 0x2c66; 0x2c68, 0x2c68; 0x2c6a, 0x2c6a; 0x2c6c, 0x2c6c; 0x2c71, 0x2c71; 0x2c73, 0x2c74; 0x2c76, 0x2c7b; 0x2c81, 0x2c81; 0x2c83, 0x2c83; 0x2c85, 0x2c85; 0x2c87, 0x2c87; 0x2c89, 0x2c89; 0x2c8b, 0x2c8b; 0x2c8d, 0x2c8d; 0x2c8f, 0x2c8f; 0x2c91, 0x2c91; 0x2c93, 0x2c93; 0x2c95, 0x2c95; 0x2c97, 0x2c97; 0x2c99, 0x2c99; 0x2c9b, 0x2c9b; 0x2c9d, 0x2c9d; 0x2c9f, 0x2c9f; 0x2ca1, 0x2ca1; 0x2ca3, 0x2ca3; 0x2ca5, 0x2ca5; 0x2ca7, 0x2ca7; 0x2ca9, 0x2ca9; 0x2cab, 0x2cab; 0x2cad, 0x2cad; 0x2caf, 0x2caf; 0x2cb1, 0x2cb1; 0x2cb3, 0x2cb3; 0x2cb5, 0x2cb5; 0x2cb7, 0x2cb7; 0x2cb9, 0x2cb9; 0x2cbb, 0x2cbb; 0x2cbd, 0x2cbd; 0x2cbf, 0x2cbf; 0x2cc1, 0x2cc1; 0x2cc3, 0x2cc3; 0x2cc5, 0x2cc5; 0x2cc7, 0x2cc7; 0x2cc9, 0x2cc9; 0x2ccb, 0x2ccb; 0x2ccd, 0x2ccd; 0x2ccf, 0x2ccf; 0x2cd1, 0x2cd1; 0x2cd3, 0x2cd3; 0x2cd5, 0x2cd5; 0x2cd7, 0x2cd7; 0x2cd9, 0x2cd9; 0x2cdb, 0x2cdb; 0x2cdd, 0x2cdd; 0x2cdf, 0x2cdf; 0x2ce1, 0x2ce1; 0x2ce3, 0x2ce4; 0x2cec, 0x2cec; 0x2cee, 0x2cee; 0x2cf3, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0xa641, 0xa641; 0xa643, 0xa643; 0xa645, 0xa645; 0xa647, 0xa647; 0xa649, 0xa649; 0xa64b, 0xa64b; 0xa64d, 0xa64d; 0xa64f, 0xa64f; 0xa651, 0xa651; 0xa653, 0xa653; 0xa655, 0xa655; 0xa657, 0xa657; 0xa659, 0xa659; 0xa65b, 0xa65b; 0xa65d, 0xa65d; 0xa65f, 0xa65f; 0xa661, 0xa661; 0xa663, 0xa663; 0xa665, 0xa665; 0xa667, 0xa667; 0xa669, 0xa669; 0xa66b, 0xa66b; 0xa66d, 0xa66d; 0xa681, 0xa681; 0xa683, 0xa683; 0xa685, 0xa685; 0xa687, 0xa687; 0xa689, 0xa689; 0xa68b, 0xa68b; 0xa68d, 0xa68d; 0xa68f, 0xa68f; 0xa691, 0xa691; 0xa693, 0xa693; 0xa695, 0xa695; 0xa697, 0xa697; 0xa699, 0xa699; 0xa69b, 0xa69b; 0xa723, 0xa723; 0xa725, 0xa725; 0xa727, 0xa727; 0xa729, 0xa729; 0xa72b, 0xa72b; 0xa72d, 0xa72d; 0xa72f, 0xa731; 0xa733, 0xa733; 0xa735, 0xa735; 0xa737, 0xa737; 0xa739, 0xa739; 0xa73b, 0xa73b; 0xa73d, 0xa73d; 0xa73f, 0xa73f; 0xa741, 0xa741; 0xa743, 0xa743; 0xa745, 0xa745; 0xa747, 0xa747; 0xa749, 0xa749; 0xa74b, 0xa74b; 0xa74d, 0xa74d; 0xa74f, 0xa74f; 0xa751, 0xa751; 0xa753, 0xa753; 0xa755, 0xa755; 0xa757, 0xa757; 0xa759, 0xa759; 0xa75b, 0xa75b; 0xa75d, 0xa75d; 0xa75f, 0xa75f; 0xa761, 0xa761; 0xa763, 0xa763; 0xa765, 0xa765; 0xa767, 0xa767; 0xa769, 0xa769; 0xa76b, 0xa76b; 0xa76d, 0xa76d; 0xa76f, 0xa76f; 0xa771, 0xa778; 0xa77a, 0xa77a; 0xa77c, 0xa77c; 0xa77f, 0xa77f; 0xa781, 0xa781; 0xa783, 0xa783; 0xa785, 0xa785; 0xa787, 0xa787; 0xa78c, 0xa78c; 0xa78e, 0xa78e; 0xa791, 0xa791; 0xa793, 0xa795; 0xa797, 0xa797; 0xa799, 0xa799; 0xa79b, 0xa79b; 0xa79d, 0xa79d; 0xa79f, 0xa79f; 0xa7a1, 0xa7a1; 0xa7a3, 0xa7a3; 0xa7a5, 0xa7a5; 0xa7a7, 0xa7a7; 0xa7a9, 0xa7a9; 0xa7af, 0xa7af; 0xa7b5, 0xa7b5; 0xa7b7, 0xa7b7; 0xa7b9, 0xa7b9; 0xa7bb, 0xa7bb; 0xa7bd, 0xa7bd; 0xa7bf, 0xa7bf; 0xa7c1, 0xa7c1; 0xa7c3, 0xa7c3; 0xa7c8, 0xa7c8; 0xa7ca, 0xa7ca; 0xa7cd, 0xa7cd; 0xa7cf, 0xa7cf; 0xa7d1, 0xa7d1; 0xa7d3, 0xa7d3; 0xa7d5, 0xa7d5; 0xa7d7, 0xa7d7; 0xa7d9, 0xa7d9; 0xa7db, 0xa7db; 0xa7f6, 0xa7f6; 0xa7fa, 0xa7fa; 0xab30, 0xab5a; 0xab60, 0xab68; 0xab70, 0xabbf; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xff41, 0xff5a; 0x10428, 0x1044f; 0x104d8, 0x104fb; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x10cc0, 0x10cf2; 0x10d70, 0x10d85; 0x118c0, 0x118df; 0x16e60, 0x16e7f; 0x16ebb, 0x16ed3; 0x1d41a, 0x1d433; 0x1d44e, 0x1d454; 0x1d456, 0x1d467; 0x1d482, 0x1d49b; 0x1d4b6, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d4cf; 0x1d4ea, 0x1d503; 0x1d51e, 0x1d537; 0x1d552, 0x1d56b; 0x1d586, 0x1d59f; 0x1d5ba, 0x1d5d3; 0x1d5ee, 0x1d607; 0x1d622, 0x1d63b; 0x1d656, 0x1d66f; 0x1d68a, 0x1d6a6; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6e1; 0x1d6fc, 0x1d714; 0x1d716, 0x1d71b; 0x1d736, 0x1d74e; 0x1d750, 0x1d755; 0x1d770, 0x1d788; 0x1d78a, 0x1d78f; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7c9; 0x1d7cb, 0x1d7cb; 0x1df00, 0x1df09; 0x1df0b, 0x1df3f; 0x1df41, 0x1df47; 0x1df49, 0x1df49; 0x1df4b, 0x1df4c; 0x1df4e, 0x1df50; 0x1df52, 0x1df67; 0x1df69, 0x1df69; 0x1df6b, 0x1df6b; 0x1df6d, 0x1df6d; 0x1df6f, 0x1df71; 0x1df73, 0x1df73; 0x1df75, 0x1df75; 0x1df77, 0x1df77; 0x1df79, 0x1df79; 0x1df7b, 0x1df7b; 0x1df7d, 0x1df7d; 0x1df7f, 0x1df7f; 0x1df90, 0x1df96; 0x1e922, 0x1e943] let lm = Cset.of_list [0x2b0, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x374, 0x374; 0x37a, 0x37a; 0x558, 0x559; 0x58b, 0x58c; 0x640, 0x640; 0x6e5, 0x6e6; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x81a, 0x81a; 0x824, 0x824; 0x828, 0x828; 0x8c9, 0x8c9; 0x971, 0x971; 0xe46, 0xe46; 0xec6, 0xec6; 0x10fc, 0x10fc; 0x17d7, 0x17d7; 0x1843, 0x1843; 0x1aa7, 0x1aa7; 0x1c78, 0x1c7d; 0x1d2c, 0x1d6a; 0x1d78, 0x1d78; 0x1d9b, 0x1dbf; 0x2071, 0x2071; 0x207f, 0x207f; 0x208f, 0x209f; 0x2c7c, 0x2c7d; 0x2d6f, 0x2d6f; 0x2e2f, 0x2e2f; 0x3005, 0x3005; 0x3031, 0x3035; 0x303b, 0x303b; 0x309d, 0x309e; 0x30fc, 0x30fe; 0xa015, 0xa015; 0xa4f8, 0xa4fd; 0xa60c, 0xa60c; 0xa67f, 0xa67f; 0xa69c, 0xa69d; 0xa717, 0xa71f; 0xa770, 0xa770; 0xa788, 0xa788; 0xa7f1, 0xa7f4; 0xa7f8, 0xa7f9; 0xa9cf, 0xa9cf; 0xa9e6, 0xa9e6; 0xaa70, 0xaa70; 0xaadd, 0xaadd; 0xaaf3, 0xaaf4; 0xab5c, 0xab5f; 0xab69, 0xab69; 0xff70, 0xff70; 0xff9e, 0xff9f; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10d4e, 0x10d4e; 0x10d6f, 0x10d6f; 0x10ec5, 0x10ec5; 0x11dd9, 0x11dd9; 0x16b40, 0x16b43; 0x16d40, 0x16d42; 0x16d6b, 0x16d6c; 0x16f93, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff2, 0x16ff3; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1dfcd, 0x1dfff; 0x1e030, 0x1e06d; 0x1e137, 0x1e13d; 0x1e4eb, 0x1e4eb; 0x1e6ff, 0x1e6ff; 0x1e94b, 0x1e94b] let lo = Cset.of_list [0xaa, 0xaa; 0xba, 0xba; 0x1bb, 0x1bb; 0x1c0, 0x1c3; 0x294, 0x295; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x620, 0x63f; 0x641, 0x64a; 0x66e, 0x66f; 0x671, 0x6d3; 0x6d5, 0x6d5; 0x6ee, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x710; 0x712, 0x72f; 0x74d, 0x7a5; 0x7b1, 0x7b1; 0x7ca, 0x7ea; 0x800, 0x815; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x8a0, 0x8c8; 0x904, 0x939; 0x93d, 0x93d; 0x950, 0x950; 0x958, 0x961; 0x972, 0x980; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9bd; 0x9ce, 0x9ce; 0x9dc, 0x9dd; 0x9df, 0x9e1; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa72, 0xa74; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xabd; 0xad0, 0xad0; 0xae0, 0xae1; 0xaf9, 0xaf9; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb3d; 0xb5c, 0xb5d; 0xb5f, 0xb61; 0xb71, 0xb71; 0xb83, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbd0, 0xbd0; 0xc05, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc3d; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc61; 0xc80, 0xc80; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcbd; 0xcdc, 0xcde; 0xce0, 0xce1; 0xcf1, 0xcf2; 0xd04, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd3d; 0xd4e, 0xd4e; 0xd54, 0xd56; 0xd5f, 0xd61; 0xd7a, 0xd7f; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xe01, 0xe30; 0xe32, 0xe33; 0xe40, 0xe45; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb0; 0xeb2, 0xeb3; 0xebd, 0xebd; 0xec0, 0xec4; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf88, 0xf8c; 0x1000, 0x102a; 0x103f, 0x103f; 0x1050, 0x1055; 0x105a, 0x105d; 0x1061, 0x1061; 0x1065, 0x1066; 0x106e, 0x1070; 0x1075, 0x1081; 0x108e, 0x108e; 0x1100, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16f1, 0x16f8; 0x1700, 0x1711; 0x171f, 0x1731; 0x1740, 0x1751; 0x1760, 0x176c; 0x176e, 0x1770; 0x1780, 0x17b3; 0x17dc, 0x17dc; 0x1820, 0x1842; 0x1844, 0x1878; 0x1880, 0x1884; 0x1887, 0x18a8; 0x18aa, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a16; 0x1a20, 0x1a54; 0x1b05, 0x1b33; 0x1b45, 0x1b4c; 0x1b83, 0x1ba0; 0x1bae, 0x1baf; 0x1bba, 0x1be5; 0x1c00, 0x1c23; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c77; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x2135, 0x2138; 0x2d30, 0x2d67; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x3006, 0x3006; 0x303c, 0x303c; 0x3041, 0x3096; 0x309f, 0x309f; 0x30a1, 0x30fa; 0x30ff, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa014; 0xa016, 0xa48c; 0xa4d0, 0xa4f7; 0xa500, 0xa60b; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa66e, 0xa66e; 0xa6a0, 0xa6e5; 0xa78f, 0xa78f; 0xa7f7, 0xa7f7; 0xa7fb, 0xa801; 0xa803, 0xa805; 0xa807, 0xa80a; 0xa80c, 0xa822; 0xa840, 0xa873; 0xa882, 0xa8b3; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8fe; 0xa90a, 0xa925; 0xa930, 0xa946; 0xa960, 0xa97c; 0xa984, 0xa9b2; 0xa9e0, 0xa9e4; 0xa9e7, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa28; 0xaa40, 0xaa42; 0xaa44, 0xaa4b; 0xaa60, 0xaa6f; 0xaa71, 0xaa76; 0xaa7a, 0xaa7a; 0xaa7e, 0xaaaf; 0xaab1, 0xaab1; 0xaab5, 0xaab6; 0xaab9, 0xaabd; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadc; 0xaae0, 0xaaea; 0xaaf2, 0xaaf2; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xabc0, 0xabe2; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb1d, 0xfb1d; 0xfb1f, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff66, 0xff6f; 0xff71, 0xff9d; 0xffa0, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x10340; 0x10342, 0x10349; 0x10350, 0x10375; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x10450, 0x1049d; 0x10500, 0x10527; 0x10530, 0x10563; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a00; 0x10a10, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10d00, 0x10d23; 0x10d4a, 0x10d4d; 0x10d4f, 0x10d4f; 0x10e80, 0x10ea9; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec4; 0x10ec6, 0x10ec7; 0x10ed9, 0x10eee; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11003, 0x11037; 0x11071, 0x11072; 0x11075, 0x11075; 0x11083, 0x110af; 0x110d0, 0x110e8; 0x11103, 0x11126; 0x11144, 0x11144; 0x11147, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11183, 0x111b2; 0x111c1, 0x111c4; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x1122b; 0x1123f, 0x11240; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112de; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x1133d; 0x11350, 0x11350; 0x1135d, 0x11361; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113b7; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11434; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114af; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115ae; 0x115d8, 0x115db; 0x11600, 0x1162f; 0x11644, 0x11644; 0x11680, 0x116aa; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x11740, 0x11746; 0x11800, 0x1182b; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x1192f; 0x1193f, 0x1193f; 0x11941, 0x11941; 0x119a0, 0x119a7; 0x119aa, 0x119d0; 0x119e1, 0x119e1; 0x119e3, 0x119e3; 0x11a00, 0x11a00; 0x11a0b, 0x11a32; 0x11a3a, 0x11a3a; 0x11a50, 0x11a50; 0x11a5c, 0x11a89; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b0a, 0x11b0a; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c2e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d30; 0x11d46, 0x11d46; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d89; 0x11d98, 0x11d98; 0x11db0, 0x11dd8; 0x11dda, 0x11ddb; 0x11df1, 0x11df1; 0x11ee0, 0x11ef2; 0x11f02, 0x11f02; 0x11f04, 0x11f10; 0x11f12, 0x11f33; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12480, 0x12543; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1611d; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d43, 0x16d6a; 0x16f00, 0x16f4a; 0x16f50, 0x16f50; 0x17000, 0x18cda; 0x18cff, 0x18d20; 0x18d80, 0x18df2; 0x18e00, 0x19191; 0x191a0, 0x191d2; 0x1b000, 0x1b128; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b168; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1df0a, 0x1df0a; 0x1df80, 0x1df81; 0x1e100, 0x1e12c; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4ea; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6e2; 0x1e6e4, 0x1e6e5; 0x1e6e7, 0x1e6ed; 0x1e6f0, 0x1e6f4; 0x1e6fe, 0x1e6fe; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x20000, 0x2a6df; 0x2a700, 0x2b81e; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0x3d000, 0x3fc3f] let lt = Cset.of_list [0x1c5, 0x1c5; 0x1c8, 0x1c8; 0x1cb, 0x1cb; 0x1f2, 0x1f2; 0x1f88, 0x1f8f; 0x1f98, 0x1f9f; 0x1fa8, 0x1faf; 0x1fbc, 0x1fbc; 0x1fcc, 0x1fcc; 0x1ffc, 0x1ffc] let lu = Cset.of_list [0x41, 0x5a; 0xc0, 0xd6; 0xd8, 0xde; 0x100, 0x100; 0x102, 0x102; 0x104, 0x104; 0x106, 0x106; 0x108, 0x108; 0x10a, 0x10a; 0x10c, 0x10c; 0x10e, 0x10e; 0x110, 0x110; 0x112, 0x112; 0x114, 0x114; 0x116, 0x116; 0x118, 0x118; 0x11a, 0x11a; 0x11c, 0x11c; 0x11e, 0x11e; 0x120, 0x120; 0x122, 0x122; 0x124, 0x124; 0x126, 0x126; 0x128, 0x128; 0x12a, 0x12a; 0x12c, 0x12c; 0x12e, 0x12e; 0x130, 0x130; 0x132, 0x132; 0x134, 0x134; 0x136, 0x136; 0x139, 0x139; 0x13b, 0x13b; 0x13d, 0x13d; 0x13f, 0x13f; 0x141, 0x141; 0x143, 0x143; 0x145, 0x145; 0x147, 0x147; 0x14a, 0x14a; 0x14c, 0x14c; 0x14e, 0x14e; 0x150, 0x150; 0x152, 0x152; 0x154, 0x154; 0x156, 0x156; 0x158, 0x158; 0x15a, 0x15a; 0x15c, 0x15c; 0x15e, 0x15e; 0x160, 0x160; 0x162, 0x162; 0x164, 0x164; 0x166, 0x166; 0x168, 0x168; 0x16a, 0x16a; 0x16c, 0x16c; 0x16e, 0x16e; 0x170, 0x170; 0x172, 0x172; 0x174, 0x174; 0x176, 0x176; 0x178, 0x179; 0x17b, 0x17b; 0x17d, 0x17d; 0x181, 0x182; 0x184, 0x184; 0x186, 0x187; 0x189, 0x18b; 0x18e, 0x191; 0x193, 0x194; 0x196, 0x198; 0x19c, 0x19d; 0x19f, 0x1a0; 0x1a2, 0x1a2; 0x1a4, 0x1a4; 0x1a6, 0x1a7; 0x1a9, 0x1a9; 0x1ac, 0x1ac; 0x1ae, 0x1af; 0x1b1, 0x1b3; 0x1b5, 0x1b5; 0x1b7, 0x1b8; 0x1bc, 0x1bc; 0x1c4, 0x1c4; 0x1c7, 0x1c7; 0x1ca, 0x1ca; 0x1cd, 0x1cd; 0x1cf, 0x1cf; 0x1d1, 0x1d1; 0x1d3, 0x1d3; 0x1d5, 0x1d5; 0x1d7, 0x1d7; 0x1d9, 0x1d9; 0x1db, 0x1db; 0x1de, 0x1de; 0x1e0, 0x1e0; 0x1e2, 0x1e2; 0x1e4, 0x1e4; 0x1e6, 0x1e6; 0x1e8, 0x1e8; 0x1ea, 0x1ea; 0x1ec, 0x1ec; 0x1ee, 0x1ee; 0x1f1, 0x1f1; 0x1f4, 0x1f4; 0x1f6, 0x1f8; 0x1fa, 0x1fa; 0x1fc, 0x1fc; 0x1fe, 0x1fe; 0x200, 0x200; 0x202, 0x202; 0x204, 0x204; 0x206, 0x206; 0x208, 0x208; 0x20a, 0x20a; 0x20c, 0x20c; 0x20e, 0x20e; 0x210, 0x210; 0x212, 0x212; 0x214, 0x214; 0x216, 0x216; 0x218, 0x218; 0x21a, 0x21a; 0x21c, 0x21c; 0x21e, 0x21e; 0x220, 0x220; 0x222, 0x222; 0x224, 0x224; 0x226, 0x226; 0x228, 0x228; 0x22a, 0x22a; 0x22c, 0x22c; 0x22e, 0x22e; 0x230, 0x230; 0x232, 0x232; 0x23a, 0x23b; 0x23d, 0x23e; 0x241, 0x241; 0x243, 0x246; 0x248, 0x248; 0x24a, 0x24a; 0x24c, 0x24c; 0x24e, 0x24e; 0x370, 0x370; 0x372, 0x372; 0x376, 0x376; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x38f; 0x391, 0x3a1; 0x3a3, 0x3ab; 0x3cf, 0x3cf; 0x3d2, 0x3d4; 0x3d8, 0x3d8; 0x3da, 0x3da; 0x3dc, 0x3dc; 0x3de, 0x3de; 0x3e0, 0x3e0; 0x3e2, 0x3e2; 0x3e4, 0x3e4; 0x3e6, 0x3e6; 0x3e8, 0x3e8; 0x3ea, 0x3ea; 0x3ec, 0x3ec; 0x3ee, 0x3ee; 0x3f4, 0x3f4; 0x3f7, 0x3f7; 0x3f9, 0x3fa; 0x3fd, 0x42f; 0x460, 0x460; 0x462, 0x462; 0x464, 0x464; 0x466, 0x466; 0x468, 0x468; 0x46a, 0x46a; 0x46c, 0x46c; 0x46e, 0x46e; 0x470, 0x470; 0x472, 0x472; 0x474, 0x474; 0x476, 0x476; 0x478, 0x478; 0x47a, 0x47a; 0x47c, 0x47c; 0x47e, 0x47e; 0x480, 0x480; 0x48a, 0x48a; 0x48c, 0x48c; 0x48e, 0x48e; 0x490, 0x490; 0x492, 0x492; 0x494, 0x494; 0x496, 0x496; 0x498, 0x498; 0x49a, 0x49a; 0x49c, 0x49c; 0x49e, 0x49e; 0x4a0, 0x4a0; 0x4a2, 0x4a2; 0x4a4, 0x4a4; 0x4a6, 0x4a6; 0x4a8, 0x4a8; 0x4aa, 0x4aa; 0x4ac, 0x4ac; 0x4ae, 0x4ae; 0x4b0, 0x4b0; 0x4b2, 0x4b2; 0x4b4, 0x4b4; 0x4b6, 0x4b6; 0x4b8, 0x4b8; 0x4ba, 0x4ba; 0x4bc, 0x4bc; 0x4be, 0x4be; 0x4c0, 0x4c1; 0x4c3, 0x4c3; 0x4c5, 0x4c5; 0x4c7, 0x4c7; 0x4c9, 0x4c9; 0x4cb, 0x4cb; 0x4cd, 0x4cd; 0x4d0, 0x4d0; 0x4d2, 0x4d2; 0x4d4, 0x4d4; 0x4d6, 0x4d6; 0x4d8, 0x4d8; 0x4da, 0x4da; 0x4dc, 0x4dc; 0x4de, 0x4de; 0x4e0, 0x4e0; 0x4e2, 0x4e2; 0x4e4, 0x4e4; 0x4e6, 0x4e6; 0x4e8, 0x4e8; 0x4ea, 0x4ea; 0x4ec, 0x4ec; 0x4ee, 0x4ee; 0x4f0, 0x4f0; 0x4f2, 0x4f2; 0x4f4, 0x4f4; 0x4f6, 0x4f6; 0x4f8, 0x4f8; 0x4fa, 0x4fa; 0x4fc, 0x4fc; 0x4fe, 0x4fe; 0x500, 0x500; 0x502, 0x502; 0x504, 0x504; 0x506, 0x506; 0x508, 0x508; 0x50a, 0x50a; 0x50c, 0x50c; 0x50e, 0x50e; 0x510, 0x510; 0x512, 0x512; 0x514, 0x514; 0x516, 0x516; 0x518, 0x518; 0x51a, 0x51a; 0x51c, 0x51c; 0x51e, 0x51e; 0x520, 0x520; 0x522, 0x522; 0x524, 0x524; 0x526, 0x526; 0x528, 0x528; 0x52a, 0x52a; 0x52c, 0x52c; 0x52e, 0x52e; 0x531, 0x556; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x13a0, 0x13f5; 0x1c89, 0x1c89; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1e00, 0x1e00; 0x1e02, 0x1e02; 0x1e04, 0x1e04; 0x1e06, 0x1e06; 0x1e08, 0x1e08; 0x1e0a, 0x1e0a; 0x1e0c, 0x1e0c; 0x1e0e, 0x1e0e; 0x1e10, 0x1e10; 0x1e12, 0x1e12; 0x1e14, 0x1e14; 0x1e16, 0x1e16; 0x1e18, 0x1e18; 0x1e1a, 0x1e1a; 0x1e1c, 0x1e1c; 0x1e1e, 0x1e1e; 0x1e20, 0x1e20; 0x1e22, 0x1e22; 0x1e24, 0x1e24; 0x1e26, 0x1e26; 0x1e28, 0x1e28; 0x1e2a, 0x1e2a; 0x1e2c, 0x1e2c; 0x1e2e, 0x1e2e; 0x1e30, 0x1e30; 0x1e32, 0x1e32; 0x1e34, 0x1e34; 0x1e36, 0x1e36; 0x1e38, 0x1e38; 0x1e3a, 0x1e3a; 0x1e3c, 0x1e3c; 0x1e3e, 0x1e3e; 0x1e40, 0x1e40; 0x1e42, 0x1e42; 0x1e44, 0x1e44; 0x1e46, 0x1e46; 0x1e48, 0x1e48; 0x1e4a, 0x1e4a; 0x1e4c, 0x1e4c; 0x1e4e, 0x1e4e; 0x1e50, 0x1e50; 0x1e52, 0x1e52; 0x1e54, 0x1e54; 0x1e56, 0x1e56; 0x1e58, 0x1e58; 0x1e5a, 0x1e5a; 0x1e5c, 0x1e5c; 0x1e5e, 0x1e5e; 0x1e60, 0x1e60; 0x1e62, 0x1e62; 0x1e64, 0x1e64; 0x1e66, 0x1e66; 0x1e68, 0x1e68; 0x1e6a, 0x1e6a; 0x1e6c, 0x1e6c; 0x1e6e, 0x1e6e; 0x1e70, 0x1e70; 0x1e72, 0x1e72; 0x1e74, 0x1e74; 0x1e76, 0x1e76; 0x1e78, 0x1e78; 0x1e7a, 0x1e7a; 0x1e7c, 0x1e7c; 0x1e7e, 0x1e7e; 0x1e80, 0x1e80; 0x1e82, 0x1e82; 0x1e84, 0x1e84; 0x1e86, 0x1e86; 0x1e88, 0x1e88; 0x1e8a, 0x1e8a; 0x1e8c, 0x1e8c; 0x1e8e, 0x1e8e; 0x1e90, 0x1e90; 0x1e92, 0x1e92; 0x1e94, 0x1e94; 0x1e9e, 0x1e9e; 0x1ea0, 0x1ea0; 0x1ea2, 0x1ea2; 0x1ea4, 0x1ea4; 0x1ea6, 0x1ea6; 0x1ea8, 0x1ea8; 0x1eaa, 0x1eaa; 0x1eac, 0x1eac; 0x1eae, 0x1eae; 0x1eb0, 0x1eb0; 0x1eb2, 0x1eb2; 0x1eb4, 0x1eb4; 0x1eb6, 0x1eb6; 0x1eb8, 0x1eb8; 0x1eba, 0x1eba; 0x1ebc, 0x1ebc; 0x1ebe, 0x1ebe; 0x1ec0, 0x1ec0; 0x1ec2, 0x1ec2; 0x1ec4, 0x1ec4; 0x1ec6, 0x1ec6; 0x1ec8, 0x1ec8; 0x1eca, 0x1eca; 0x1ecc, 0x1ecc; 0x1ece, 0x1ece; 0x1ed0, 0x1ed0; 0x1ed2, 0x1ed2; 0x1ed4, 0x1ed4; 0x1ed6, 0x1ed6; 0x1ed8, 0x1ed8; 0x1eda, 0x1eda; 0x1edc, 0x1edc; 0x1ede, 0x1ede; 0x1ee0, 0x1ee0; 0x1ee2, 0x1ee2; 0x1ee4, 0x1ee4; 0x1ee6, 0x1ee6; 0x1ee8, 0x1ee8; 0x1eea, 0x1eea; 0x1eec, 0x1eec; 0x1eee, 0x1eee; 0x1ef0, 0x1ef0; 0x1ef2, 0x1ef2; 0x1ef4, 0x1ef4; 0x1ef6, 0x1ef6; 0x1ef8, 0x1ef8; 0x1efa, 0x1efa; 0x1efc, 0x1efc; 0x1efe, 0x1efe; 0x1f08, 0x1f0f; 0x1f18, 0x1f1d; 0x1f28, 0x1f2f; 0x1f38, 0x1f3f; 0x1f48, 0x1f4d; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f5f; 0x1f68, 0x1f6f; 0x1fb8, 0x1fbb; 0x1fc8, 0x1fcb; 0x1fd8, 0x1fdb; 0x1fe8, 0x1fec; 0x1ff8, 0x1ffb; 0x2102, 0x2102; 0x2107, 0x2107; 0x210b, 0x210d; 0x2110, 0x2112; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x212d; 0x2130, 0x2133; 0x213e, 0x213f; 0x2145, 0x2145; 0x2183, 0x2183; 0x2c00, 0x2c2f; 0x2c60, 0x2c60; 0x2c62, 0x2c64; 0x2c67, 0x2c67; 0x2c69, 0x2c69; 0x2c6b, 0x2c6b; 0x2c6d, 0x2c70; 0x2c72, 0x2c72; 0x2c75, 0x2c75; 0x2c7e, 0x2c80; 0x2c82, 0x2c82; 0x2c84, 0x2c84; 0x2c86, 0x2c86; 0x2c88, 0x2c88; 0x2c8a, 0x2c8a; 0x2c8c, 0x2c8c; 0x2c8e, 0x2c8e; 0x2c90, 0x2c90; 0x2c92, 0x2c92; 0x2c94, 0x2c94; 0x2c96, 0x2c96; 0x2c98, 0x2c98; 0x2c9a, 0x2c9a; 0x2c9c, 0x2c9c; 0x2c9e, 0x2c9e; 0x2ca0, 0x2ca0; 0x2ca2, 0x2ca2; 0x2ca4, 0x2ca4; 0x2ca6, 0x2ca6; 0x2ca8, 0x2ca8; 0x2caa, 0x2caa; 0x2cac, 0x2cac; 0x2cae, 0x2cae; 0x2cb0, 0x2cb0; 0x2cb2, 0x2cb2; 0x2cb4, 0x2cb4; 0x2cb6, 0x2cb6; 0x2cb8, 0x2cb8; 0x2cba, 0x2cba; 0x2cbc, 0x2cbc; 0x2cbe, 0x2cbe; 0x2cc0, 0x2cc0; 0x2cc2, 0x2cc2; 0x2cc4, 0x2cc4; 0x2cc6, 0x2cc6; 0x2cc8, 0x2cc8; 0x2cca, 0x2cca; 0x2ccc, 0x2ccc; 0x2cce, 0x2cce; 0x2cd0, 0x2cd0; 0x2cd2, 0x2cd2; 0x2cd4, 0x2cd4; 0x2cd6, 0x2cd6; 0x2cd8, 0x2cd8; 0x2cda, 0x2cda; 0x2cdc, 0x2cdc; 0x2cde, 0x2cde; 0x2ce0, 0x2ce0; 0x2ce2, 0x2ce2; 0x2ceb, 0x2ceb; 0x2ced, 0x2ced; 0x2cf2, 0x2cf2; 0xa640, 0xa640; 0xa642, 0xa642; 0xa644, 0xa644; 0xa646, 0xa646; 0xa648, 0xa648; 0xa64a, 0xa64a; 0xa64c, 0xa64c; 0xa64e, 0xa64e; 0xa650, 0xa650; 0xa652, 0xa652; 0xa654, 0xa654; 0xa656, 0xa656; 0xa658, 0xa658; 0xa65a, 0xa65a; 0xa65c, 0xa65c; 0xa65e, 0xa65e; 0xa660, 0xa660; 0xa662, 0xa662; 0xa664, 0xa664; 0xa666, 0xa666; 0xa668, 0xa668; 0xa66a, 0xa66a; 0xa66c, 0xa66c; 0xa680, 0xa680; 0xa682, 0xa682; 0xa684, 0xa684; 0xa686, 0xa686; 0xa688, 0xa688; 0xa68a, 0xa68a; 0xa68c, 0xa68c; 0xa68e, 0xa68e; 0xa690, 0xa690; 0xa692, 0xa692; 0xa694, 0xa694; 0xa696, 0xa696; 0xa698, 0xa698; 0xa69a, 0xa69a; 0xa722, 0xa722; 0xa724, 0xa724; 0xa726, 0xa726; 0xa728, 0xa728; 0xa72a, 0xa72a; 0xa72c, 0xa72c; 0xa72e, 0xa72e; 0xa732, 0xa732; 0xa734, 0xa734; 0xa736, 0xa736; 0xa738, 0xa738; 0xa73a, 0xa73a; 0xa73c, 0xa73c; 0xa73e, 0xa73e; 0xa740, 0xa740; 0xa742, 0xa742; 0xa744, 0xa744; 0xa746, 0xa746; 0xa748, 0xa748; 0xa74a, 0xa74a; 0xa74c, 0xa74c; 0xa74e, 0xa74e; 0xa750, 0xa750; 0xa752, 0xa752; 0xa754, 0xa754; 0xa756, 0xa756; 0xa758, 0xa758; 0xa75a, 0xa75a; 0xa75c, 0xa75c; 0xa75e, 0xa75e; 0xa760, 0xa760; 0xa762, 0xa762; 0xa764, 0xa764; 0xa766, 0xa766; 0xa768, 0xa768; 0xa76a, 0xa76a; 0xa76c, 0xa76c; 0xa76e, 0xa76e; 0xa779, 0xa779; 0xa77b, 0xa77b; 0xa77d, 0xa77e; 0xa780, 0xa780; 0xa782, 0xa782; 0xa784, 0xa784; 0xa786, 0xa786; 0xa78b, 0xa78b; 0xa78d, 0xa78d; 0xa790, 0xa790; 0xa792, 0xa792; 0xa796, 0xa796; 0xa798, 0xa798; 0xa79a, 0xa79a; 0xa79c, 0xa79c; 0xa79e, 0xa79e; 0xa7a0, 0xa7a0; 0xa7a2, 0xa7a2; 0xa7a4, 0xa7a4; 0xa7a6, 0xa7a6; 0xa7a8, 0xa7a8; 0xa7aa, 0xa7ae; 0xa7b0, 0xa7b4; 0xa7b6, 0xa7b6; 0xa7b8, 0xa7b8; 0xa7ba, 0xa7ba; 0xa7bc, 0xa7bc; 0xa7be, 0xa7be; 0xa7c0, 0xa7c0; 0xa7c2, 0xa7c2; 0xa7c4, 0xa7c7; 0xa7c9, 0xa7c9; 0xa7cb, 0xa7cc; 0xa7ce, 0xa7ce; 0xa7d0, 0xa7d0; 0xa7d2, 0xa7d2; 0xa7d4, 0xa7d4; 0xa7d6, 0xa7d6; 0xa7d8, 0xa7d8; 0xa7da, 0xa7da; 0xa7dc, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f5, 0xa7f5; 0xab6c, 0xab6d; 0xff21, 0xff3a; 0x10400, 0x10427; 0x104b0, 0x104d3; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10c80, 0x10cb2; 0x10d50, 0x10d65; 0x118a0, 0x118bf; 0x16e40, 0x16e5f; 0x16ea0, 0x16eb8; 0x1d400, 0x1d419; 0x1d434, 0x1d44d; 0x1d468, 0x1d481; 0x1d49c, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b5; 0x1d4d0, 0x1d4e9; 0x1d504, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d538, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d56c, 0x1d585; 0x1d5a0, 0x1d5b9; 0x1d5d4, 0x1d5ed; 0x1d608, 0x1d621; 0x1d63c, 0x1d655; 0x1d670, 0x1d689; 0x1d6a8, 0x1d6c0; 0x1d6e2, 0x1d6fa; 0x1d71c, 0x1d734; 0x1d756, 0x1d76e; 0x1d790, 0x1d7a8; 0x1d7ca, 0x1d7ca; 0x1df40, 0x1df40; 0x1df48, 0x1df48; 0x1df4a, 0x1df4a; 0x1df4d, 0x1df4d; 0x1df51, 0x1df51; 0x1df68, 0x1df68; 0x1df6a, 0x1df6a; 0x1df6c, 0x1df6c; 0x1df6e, 0x1df6e; 0x1df72, 0x1df72; 0x1df74, 0x1df74; 0x1df76, 0x1df76; 0x1df78, 0x1df78; 0x1df7a, 0x1df7a; 0x1df7c, 0x1df7c; 0x1df7e, 0x1df7e; 0x1e900, 0x1e921] let mc = Cset.of_list [0x903, 0x903; 0x93b, 0x93b; 0x93e, 0x940; 0x949, 0x94c; 0x94e, 0x94f; 0x982, 0x983; 0x9be, 0x9c0; 0x9c7, 0x9c8; 0x9cb, 0x9cc; 0x9d7, 0x9d7; 0xa03, 0xa03; 0xa3e, 0xa40; 0xa83, 0xa83; 0xabe, 0xac0; 0xac9, 0xac9; 0xacb, 0xacc; 0xb02, 0xb03; 0xb3e, 0xb3e; 0xb40, 0xb40; 0xb47, 0xb48; 0xb4b, 0xb4c; 0xb57, 0xb57; 0xbbe, 0xbbf; 0xbc1, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcc; 0xbd7, 0xbd7; 0xc01, 0xc03; 0xc41, 0xc44; 0xc82, 0xc83; 0xcbe, 0xcbe; 0xcc0, 0xcc4; 0xcc7, 0xcc8; 0xcca, 0xccb; 0xcd5, 0xcd6; 0xcf3, 0xcf3; 0xd02, 0xd03; 0xd3e, 0xd40; 0xd46, 0xd48; 0xd4a, 0xd4c; 0xd57, 0xd57; 0xd82, 0xd83; 0xdcf, 0xdd1; 0xdd8, 0xddf; 0xdf2, 0xdf3; 0xf3e, 0xf3f; 0xf7f, 0xf7f; 0x102b, 0x102c; 0x1031, 0x1031; 0x1038, 0x1038; 0x103b, 0x103c; 0x1056, 0x1057; 0x1062, 0x1064; 0x1067, 0x106d; 0x1083, 0x1084; 0x1087, 0x108c; 0x108f, 0x108f; 0x109a, 0x109c; 0x1715, 0x1715; 0x1734, 0x1734; 0x17b6, 0x17b6; 0x17be, 0x17c5; 0x17c7, 0x17c8; 0x1923, 0x1926; 0x1929, 0x192b; 0x1930, 0x1931; 0x1933, 0x1938; 0x1a19, 0x1a1a; 0x1a55, 0x1a55; 0x1a57, 0x1a57; 0x1a61, 0x1a61; 0x1a63, 0x1a64; 0x1a6d, 0x1a72; 0x1b04, 0x1b04; 0x1b35, 0x1b35; 0x1b3b, 0x1b3b; 0x1b3d, 0x1b41; 0x1b43, 0x1b44; 0x1b82, 0x1b82; 0x1ba1, 0x1ba1; 0x1ba6, 0x1ba7; 0x1baa, 0x1baa; 0x1be7, 0x1be7; 0x1bea, 0x1bec; 0x1bee, 0x1bee; 0x1bf2, 0x1bf3; 0x1c24, 0x1c2b; 0x1c34, 0x1c35; 0x1ce1, 0x1ce1; 0x1cf7, 0x1cf7; 0x302e, 0x302f; 0xa823, 0xa824; 0xa827, 0xa827; 0xa880, 0xa881; 0xa8b4, 0xa8c3; 0xa952, 0xa953; 0xa983, 0xa983; 0xa9b4, 0xa9b5; 0xa9ba, 0xa9bb; 0xa9be, 0xa9c0; 0xaa2f, 0xaa30; 0xaa33, 0xaa34; 0xaa4d, 0xaa4d; 0xaa7b, 0xaa7b; 0xaa7d, 0xaa7d; 0xaaeb, 0xaaeb; 0xaaee, 0xaaef; 0xaaf5, 0xaaf5; 0xabe3, 0xabe4; 0xabe6, 0xabe7; 0xabe9, 0xabea; 0xabec, 0xabec; 0x11000, 0x11000; 0x11002, 0x11002; 0x11082, 0x11082; 0x110b0, 0x110b2; 0x110b7, 0x110b8; 0x1112c, 0x1112c; 0x11145, 0x11146; 0x11182, 0x11182; 0x111b3, 0x111b5; 0x111bf, 0x111c0; 0x111ce, 0x111ce; 0x1122c, 0x1122e; 0x11232, 0x11233; 0x11235, 0x11235; 0x112e0, 0x112e2; 0x11302, 0x11303; 0x1133e, 0x1133f; 0x11341, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134d; 0x11357, 0x11357; 0x11362, 0x11363; 0x113b8, 0x113ba; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113cd; 0x113cf, 0x113cf; 0x11435, 0x11437; 0x11440, 0x11441; 0x11445, 0x11445; 0x114b0, 0x114b2; 0x114b9, 0x114b9; 0x114bb, 0x114be; 0x114c1, 0x114c1; 0x115af, 0x115b1; 0x115b8, 0x115bb; 0x115be, 0x115be; 0x11630, 0x11632; 0x1163b, 0x1163c; 0x1163e, 0x1163e; 0x116ac, 0x116ac; 0x116ae, 0x116af; 0x116b6, 0x116b6; 0x1171e, 0x1171e; 0x11720, 0x11721; 0x11726, 0x11726; 0x1182c, 0x1182e; 0x11838, 0x11838; 0x11930, 0x11935; 0x11937, 0x11938; 0x1193d, 0x1193d; 0x11940, 0x11940; 0x11942, 0x11942; 0x119d1, 0x119d3; 0x119dc, 0x119df; 0x119e4, 0x119e4; 0x11a39, 0x11a39; 0x11a57, 0x11a58; 0x11a97, 0x11a97; 0x11b61, 0x11b61; 0x11b65, 0x11b65; 0x11b67, 0x11b67; 0x11c2f, 0x11c2f; 0x11c3e, 0x11c3e; 0x11ca9, 0x11ca9; 0x11cb1, 0x11cb1; 0x11cb4, 0x11cb4; 0x11d8a, 0x11d8e; 0x11d93, 0x11d94; 0x11d96, 0x11d96; 0x11ef5, 0x11ef6; 0x11f03, 0x11f03; 0x11f34, 0x11f35; 0x11f3e, 0x11f3f; 0x11f41, 0x11f41; 0x1612a, 0x1612c; 0x16f51, 0x16f87; 0x16ff0, 0x16ff1; 0x1d165, 0x1d166; 0x1d16d, 0x1d172; 0x1d250, 0x1d252; 0x1d25f, 0x1d25f; 0x1d280, 0x1d281] let me = Cset.of_list [0x488, 0x489; 0x1abe, 0x1abe; 0x20dd, 0x20e0; 0x20e2, 0x20e4; 0xa670, 0xa672] let mn = Cset.of_list [0x300, 0x36f; 0x483, 0x487; 0x591, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c9; 0x610, 0x61a; 0x64b, 0x65f; 0x670, 0x670; 0x6d6, 0x6dc; 0x6df, 0x6e4; 0x6e7, 0x6e8; 0x6ea, 0x6ed; 0x711, 0x711; 0x730, 0x74a; 0x7a6, 0x7b0; 0x7eb, 0x7f3; 0x7fd, 0x7fd; 0x816, 0x819; 0x81b, 0x823; 0x825, 0x827; 0x829, 0x82d; 0x859, 0x85b; 0x897, 0x89f; 0x8ca, 0x8e1; 0x8e3, 0x902; 0x93a, 0x93a; 0x93c, 0x93c; 0x941, 0x948; 0x94d, 0x94d; 0x951, 0x957; 0x962, 0x963; 0x981, 0x981; 0x9bc, 0x9bc; 0x9c1, 0x9c4; 0x9cd, 0x9cd; 0x9e2, 0x9e3; 0x9fe, 0x9fe; 0xa01, 0xa02; 0xa3c, 0xa3c; 0xa41, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4d; 0xa51, 0xa51; 0xa70, 0xa71; 0xa75, 0xa75; 0xa81, 0xa82; 0xabc, 0xabc; 0xac1, 0xac5; 0xac7, 0xac8; 0xacd, 0xacd; 0xae2, 0xae3; 0xafa, 0xaff; 0xb01, 0xb01; 0xb3c, 0xb3c; 0xb3f, 0xb3f; 0xb41, 0xb44; 0xb4d, 0xb4d; 0xb53, 0xb56; 0xb62, 0xb63; 0xb82, 0xb82; 0xbc0, 0xbc0; 0xbcd, 0xbcd; 0xc00, 0xc00; 0xc04, 0xc04; 0xc3c, 0xc3c; 0xc3e, 0xc40; 0xc46, 0xc48; 0xc4a, 0xc4d; 0xc55, 0xc56; 0xc62, 0xc63; 0xc81, 0xc81; 0xcbc, 0xcbc; 0xcbf, 0xcbf; 0xcc6, 0xcc6; 0xccc, 0xccd; 0xce2, 0xce3; 0xd00, 0xd01; 0xd3b, 0xd3c; 0xd41, 0xd44; 0xd4d, 0xd4d; 0xd62, 0xd63; 0xd81, 0xd81; 0xdca, 0xdca; 0xdd2, 0xdd4; 0xdd6, 0xdd6; 0xe31, 0xe31; 0xe34, 0xe3a; 0xe47, 0xe4e; 0xeb1, 0xeb1; 0xeb4, 0xebc; 0xec8, 0xece; 0xf18, 0xf19; 0xf35, 0xf35; 0xf37, 0xf37; 0xf39, 0xf39; 0xf71, 0xf7e; 0xf80, 0xf84; 0xf86, 0xf87; 0xf8d, 0xf97; 0xf99, 0xfbc; 0xfc6, 0xfc6; 0x102d, 0x1030; 0x1032, 0x1037; 0x1039, 0x103a; 0x103d, 0x103e; 0x1058, 0x1059; 0x105e, 0x1060; 0x1071, 0x1074; 0x1082, 0x1082; 0x1085, 0x1086; 0x108d, 0x108d; 0x109d, 0x109d; 0x135d, 0x135f; 0x1712, 0x1714; 0x1732, 0x1733; 0x1752, 0x1753; 0x1772, 0x1773; 0x17b4, 0x17b5; 0x17b7, 0x17bd; 0x17c6, 0x17c6; 0x17c9, 0x17d3; 0x17dd, 0x17dd; 0x180b, 0x180d; 0x180f, 0x180f; 0x1885, 0x1886; 0x18a9, 0x18a9; 0x1920, 0x1922; 0x1927, 0x1928; 0x1932, 0x1932; 0x1939, 0x193b; 0x1a17, 0x1a18; 0x1a1b, 0x1a1b; 0x1a56, 0x1a56; 0x1a58, 0x1a5e; 0x1a60, 0x1a60; 0x1a62, 0x1a62; 0x1a65, 0x1a6c; 0x1a73, 0x1a7c; 0x1a7f, 0x1a7f; 0x1ab0, 0x1abd; 0x1abf, 0x1af0; 0x1b00, 0x1b03; 0x1b34, 0x1b34; 0x1b36, 0x1b3a; 0x1b3c, 0x1b3c; 0x1b42, 0x1b42; 0x1b6b, 0x1b73; 0x1b80, 0x1b81; 0x1ba2, 0x1ba5; 0x1ba8, 0x1ba9; 0x1bab, 0x1bad; 0x1be6, 0x1be6; 0x1be8, 0x1be9; 0x1bed, 0x1bed; 0x1bef, 0x1bf1; 0x1c2c, 0x1c33; 0x1c36, 0x1c37; 0x1cd0, 0x1cd2; 0x1cd4, 0x1ce0; 0x1ce2, 0x1ce8; 0x1ced, 0x1ced; 0x1cf4, 0x1cf4; 0x1cf8, 0x1cf9; 0x1dc0, 0x1dff; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20f0; 0x2cef, 0x2cf1; 0x2d7f, 0x2d7f; 0x2de0, 0x2dff; 0x302a, 0x302d; 0x3099, 0x309a; 0xa66f, 0xa66f; 0xa674, 0xa67d; 0xa69e, 0xa69f; 0xa6f0, 0xa6f1; 0xa802, 0xa802; 0xa806, 0xa806; 0xa80b, 0xa80b; 0xa825, 0xa826; 0xa82c, 0xa82c; 0xa8c4, 0xa8c5; 0xa8e0, 0xa8f1; 0xa8ff, 0xa8ff; 0xa926, 0xa92d; 0xa947, 0xa951; 0xa980, 0xa982; 0xa9b3, 0xa9b3; 0xa9b6, 0xa9b9; 0xa9bc, 0xa9bd; 0xa9e5, 0xa9e5; 0xaa29, 0xaa2e; 0xaa31, 0xaa32; 0xaa35, 0xaa36; 0xaa43, 0xaa43; 0xaa4c, 0xaa4c; 0xaa7c, 0xaa7c; 0xaab0, 0xaab0; 0xaab2, 0xaab4; 0xaab7, 0xaab8; 0xaabe, 0xaabf; 0xaac1, 0xaac1; 0xaaec, 0xaaed; 0xaaf6, 0xaaf6; 0xabe5, 0xabe5; 0xabe8, 0xabe8; 0xabed, 0xabed; 0xfb1e, 0xfb1e; 0xfe00, 0xfe0f; 0xfe20, 0xfe2f; 0x101fd, 0x101fd; 0x102e0, 0x102e0; 0x10376, 0x1037a; 0x10a01, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a0f; 0x10a38, 0x10a3a; 0x10a3f, 0x10a3f; 0x10ae5, 0x10ae6; 0x10d24, 0x10d27; 0x10d69, 0x10d6d; 0x10eab, 0x10eac; 0x10ecb, 0x10ecf; 0x10ef0, 0x10eff; 0x10f46, 0x10f50; 0x10f82, 0x10f85; 0x11001, 0x11001; 0x11038, 0x11046; 0x11070, 0x11070; 0x11073, 0x11074; 0x1107f, 0x11081; 0x110b3, 0x110b6; 0x110b9, 0x110ba; 0x110c2, 0x110c2; 0x11100, 0x11102; 0x11127, 0x1112b; 0x1112d, 0x11134; 0x11173, 0x11173; 0x11180, 0x11181; 0x111b6, 0x111be; 0x111c9, 0x111cc; 0x111cf, 0x111cf; 0x1122f, 0x11231; 0x11234, 0x11234; 0x11236, 0x11237; 0x1123e, 0x1123e; 0x11241, 0x11241; 0x112df, 0x112df; 0x112e3, 0x112ea; 0x11300, 0x11301; 0x1133b, 0x1133c; 0x11340, 0x11340; 0x11366, 0x1136c; 0x11370, 0x11374; 0x113bb, 0x113c0; 0x113ce, 0x113ce; 0x113d0, 0x113d0; 0x113d2, 0x113d2; 0x113e1, 0x113e2; 0x11438, 0x1143f; 0x11442, 0x11444; 0x11446, 0x11446; 0x1145e, 0x1145e; 0x114b3, 0x114b8; 0x114ba, 0x114ba; 0x114bf, 0x114c0; 0x114c2, 0x114c3; 0x115b2, 0x115b5; 0x115bc, 0x115bd; 0x115bf, 0x115c0; 0x115dc, 0x115dd; 0x11633, 0x1163a; 0x1163d, 0x1163d; 0x1163f, 0x11640; 0x116ab, 0x116ab; 0x116ad, 0x116ad; 0x116b0, 0x116b5; 0x116b7, 0x116b7; 0x1171d, 0x1171d; 0x1171f, 0x1171f; 0x11722, 0x11725; 0x11727, 0x1172b; 0x1182f, 0x11837; 0x11839, 0x1183a; 0x1193b, 0x1193c; 0x1193e, 0x1193e; 0x11943, 0x11943; 0x119d4, 0x119d7; 0x119da, 0x119db; 0x119e0, 0x119e0; 0x11a01, 0x11a0a; 0x11a33, 0x11a38; 0x11a3b, 0x11a3e; 0x11a47, 0x11a47; 0x11a51, 0x11a56; 0x11a59, 0x11a5b; 0x11a8a, 0x11a96; 0x11a98, 0x11a99; 0x11b60, 0x11b60; 0x11b62, 0x11b64; 0x11b66, 0x11b66; 0x11c30, 0x11c36; 0x11c38, 0x11c3d; 0x11c3f, 0x11c3f; 0x11c92, 0x11ca7; 0x11caa, 0x11cb0; 0x11cb2, 0x11cb3; 0x11cb5, 0x11cb6; 0x11d31, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d45; 0x11d47, 0x11d47; 0x11d90, 0x11d91; 0x11d95, 0x11d95; 0x11d97, 0x11d97; 0x11df0, 0x11df0; 0x11ef3, 0x11ef4; 0x11f00, 0x11f01; 0x11f36, 0x11f3a; 0x11f40, 0x11f40; 0x11f42, 0x11f42; 0x11f5a, 0x11f5a; 0x13440, 0x13440; 0x13447, 0x13455; 0x1611e, 0x16129; 0x1612d, 0x1612f; 0x16af0, 0x16af4; 0x16b30, 0x16b36; 0x16f4f, 0x16f4f; 0x16f8f, 0x16f92; 0x16fe4, 0x16fe4; 0x1bc9d, 0x1bc9e; 0x1cf00, 0x1cf2d; 0x1cf30, 0x1cf46; 0x1d127, 0x1d128; 0x1d167, 0x1d169; 0x1d17b, 0x1d182; 0x1d185, 0x1d18b; 0x1d1aa, 0x1d1ad; 0x1d242, 0x1d244; 0x1d25b, 0x1d25c; 0x1da00, 0x1da36; 0x1da3b, 0x1da6c; 0x1da75, 0x1da75; 0x1da84, 0x1da84; 0x1da9b, 0x1da9f; 0x1daa1, 0x1daaf; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e08f, 0x1e08f; 0x1e130, 0x1e136; 0x1e2ae, 0x1e2ae; 0x1e2ec, 0x1e2ef; 0x1e4ec, 0x1e4ef; 0x1e5ee, 0x1e5ef; 0x1e6e3, 0x1e6e3; 0x1e6e6, 0x1e6e6; 0x1e6ee, 0x1e6ef; 0x1e6f5, 0x1e6f5; 0x1e8d0, 0x1e8d6; 0x1e944, 0x1e94a; 0xe0100, 0xe01ef] let nd = Cset.of_list [0x30, 0x39; 0x660, 0x669; 0x6f0, 0x6f9; 0x7c0, 0x7c9; 0x966, 0x96f; 0x9e6, 0x9ef; 0xa66, 0xa6f; 0xae6, 0xaef; 0xb66, 0xb6f; 0xbe6, 0xbef; 0xc66, 0xc6f; 0xce6, 0xcef; 0xd66, 0xd6f; 0xde6, 0xdef; 0xe50, 0xe59; 0xed0, 0xed9; 0xf20, 0xf29; 0x1040, 0x1049; 0x1090, 0x1099; 0x17e0, 0x17e9; 0x1810, 0x1819; 0x1946, 0x194f; 0x19d0, 0x19d9; 0x1a80, 0x1a89; 0x1a90, 0x1a99; 0x1b50, 0x1b59; 0x1bb0, 0x1bb9; 0x1c40, 0x1c49; 0x1c50, 0x1c59; 0xa620, 0xa629; 0xa8d0, 0xa8d9; 0xa900, 0xa909; 0xa9d0, 0xa9d9; 0xa9f0, 0xa9f9; 0xaa50, 0xaa59; 0xabf0, 0xabf9; 0xff10, 0xff19; 0x104a0, 0x104a9; 0x10d30, 0x10d39; 0x10d40, 0x10d49; 0x11066, 0x1106f; 0x110f0, 0x110f9; 0x11136, 0x1113f; 0x111d0, 0x111d9; 0x112f0, 0x112f9; 0x11450, 0x11459; 0x114d0, 0x114d9; 0x11650, 0x11659; 0x116c0, 0x116c9; 0x116d0, 0x116e3; 0x11730, 0x11739; 0x118e0, 0x118e9; 0x11950, 0x11959; 0x11bf0, 0x11bf9; 0x11c50, 0x11c59; 0x11d50, 0x11d59; 0x11da0, 0x11da9; 0x11de0, 0x11de9; 0x11f50, 0x11f59; 0x16130, 0x16139; 0x16a60, 0x16a69; 0x16ac0, 0x16ac9; 0x16b50, 0x16b59; 0x16d70, 0x16d79; 0x1ccf0, 0x1ccf9; 0x1d7ce, 0x1d7ff; 0x1e140, 0x1e149; 0x1e2f0, 0x1e2f9; 0x1e4f0, 0x1e4f9; 0x1e5f1, 0x1e5fa; 0x1e950, 0x1e959; 0x1fbf0, 0x1fbf9] let nl = Cset.of_list [0x16ee, 0x16f0; 0x2160, 0x2182; 0x2185, 0x2188; 0x3007, 0x3007; 0x3021, 0x3029; 0x3038, 0x303a; 0xa6e6, 0xa6ef; 0x10140, 0x10174; 0x10341, 0x10341; 0x1034a, 0x1034a; 0x103d1, 0x103d5; 0x12400, 0x1246f; 0x12475, 0x1247f; 0x12550, 0x12686; 0x16ff4, 0x16ff6] let no = Cset.of_list [0xb2, 0xb3; 0xb9, 0xb9; 0xbc, 0xbe; 0x9f4, 0x9f9; 0xb72, 0xb77; 0xbf0, 0xbf2; 0xc78, 0xc7e; 0xd58, 0xd5e; 0xd70, 0xd78; 0xf2a, 0xf33; 0x1369, 0x137c; 0x17f0, 0x17f9; 0x19da, 0x19da; 0x2070, 0x2070; 0x2074, 0x2079; 0x2080, 0x2089; 0x2150, 0x215f; 0x2189, 0x2189; 0x2460, 0x249b; 0x24ea, 0x24ff; 0x2776, 0x2793; 0x2cfd, 0x2cfd; 0x3192, 0x3195; 0x3220, 0x3229; 0x3248, 0x324f; 0x3251, 0x325f; 0x3280, 0x3289; 0x32b1, 0x32bf; 0xa830, 0xa835; 0x10107, 0x10133; 0x10175, 0x10178; 0x1018a, 0x1018b; 0x102e1, 0x102fb; 0x10320, 0x10323; 0x10858, 0x1085f; 0x10879, 0x1087f; 0x108a7, 0x108af; 0x108fb, 0x108ff; 0x10916, 0x1091b; 0x109bc, 0x109bd; 0x109c0, 0x109cf; 0x109d2, 0x109ff; 0x10a40, 0x10a48; 0x10a7d, 0x10a7e; 0x10a9d, 0x10a9f; 0x10aeb, 0x10aef; 0x10b58, 0x10b5f; 0x10b78, 0x10b7f; 0x10ba9, 0x10baf; 0x10cfa, 0x10cff; 0x10e60, 0x10e7e; 0x10f1d, 0x10f26; 0x10f51, 0x10f54; 0x10fc5, 0x10fcb; 0x11052, 0x11065; 0x111e1, 0x111f4; 0x1173a, 0x1173b; 0x118ea, 0x118f2; 0x11c5a, 0x11c6c; 0x11fc0, 0x11fd4; 0x16b5b, 0x16b61; 0x16e80, 0x16e96; 0x1d2c0, 0x1d2d3; 0x1d2e0, 0x1d2f3; 0x1d360, 0x1d378; 0x1e8c7, 0x1e8cf; 0x1ec71, 0x1ecab; 0x1ecad, 0x1ecaf; 0x1ecb1, 0x1ecb4; 0x1ed01, 0x1ed2d; 0x1ed2f, 0x1ed3d; 0x1f100, 0x1f10c] let pc = Cset.of_list [0x5f, 0x5f; 0x203f, 0x2040; 0x2054, 0x2054; 0xfe33, 0xfe34; 0xfe4d, 0xfe4f; 0xff3f, 0xff3f] let pd = Cset.of_list [0x2d, 0x2d; 0x58a, 0x58a; 0x5be, 0x5be; 0x1400, 0x1400; 0x1806, 0x1806; 0x2010, 0x2015; 0x2e17, 0x2e17; 0x2e1a, 0x2e1a; 0x2e3a, 0x2e3b; 0x2e40, 0x2e40; 0x2e5d, 0x2e5d; 0x301c, 0x301c; 0x3030, 0x3030; 0x30a0, 0x30a0; 0xfe31, 0xfe32; 0xfe58, 0xfe58; 0xfe63, 0xfe63; 0xff0d, 0xff0d; 0x10d6e, 0x10d6e; 0x10ead, 0x10ead] let pe = Cset.of_list [0x29, 0x29; 0x5d, 0x5d; 0x7d, 0x7d; 0xf3b, 0xf3b; 0xf3d, 0xf3d; 0x169c, 0x169c; 0x2046, 0x2046; 0x207e, 0x207e; 0x208e, 0x208e; 0x2309, 0x2309; 0x230b, 0x230b; 0x232a, 0x232a; 0x2769, 0x2769; 0x276b, 0x276b; 0x276d, 0x276d; 0x276f, 0x276f; 0x2771, 0x2771; 0x2773, 0x2773; 0x2775, 0x2775; 0x27c6, 0x27c6; 0x27e7, 0x27e7; 0x27e9, 0x27e9; 0x27eb, 0x27eb; 0x27ed, 0x27ed; 0x27ef, 0x27ef; 0x2984, 0x2984; 0x2986, 0x2986; 0x2988, 0x2988; 0x298a, 0x298a; 0x298c, 0x298c; 0x298e, 0x298e; 0x2990, 0x2990; 0x2992, 0x2992; 0x2994, 0x2994; 0x2996, 0x2996; 0x2998, 0x2998; 0x29d9, 0x29d9; 0x29db, 0x29db; 0x29fd, 0x29fd; 0x2e23, 0x2e23; 0x2e25, 0x2e25; 0x2e27, 0x2e27; 0x2e29, 0x2e29; 0x2e56, 0x2e56; 0x2e58, 0x2e58; 0x2e5a, 0x2e5a; 0x2e5c, 0x2e5c; 0x2e63, 0x2e63; 0x3009, 0x3009; 0x300b, 0x300b; 0x300d, 0x300d; 0x300f, 0x300f; 0x3011, 0x3011; 0x3015, 0x3015; 0x3017, 0x3017; 0x3019, 0x3019; 0x301b, 0x301b; 0x301e, 0x301f; 0xfd3e, 0xfd3e; 0xfe18, 0xfe18; 0xfe36, 0xfe36; 0xfe38, 0xfe38; 0xfe3a, 0xfe3a; 0xfe3c, 0xfe3c; 0xfe3e, 0xfe3e; 0xfe40, 0xfe40; 0xfe42, 0xfe42; 0xfe44, 0xfe44; 0xfe48, 0xfe48; 0xfe5a, 0xfe5a; 0xfe5c, 0xfe5c; 0xfe5e, 0xfe5e; 0xff09, 0xff09; 0xff3d, 0xff3d; 0xff5d, 0xff5d; 0xff60, 0xff60; 0xff63, 0xff63] let pf = Cset.of_list [0xbb, 0xbb; 0x2019, 0x2019; 0x201d, 0x201d; 0x203a, 0x203a; 0x2e03, 0x2e03; 0x2e05, 0x2e05; 0x2e0a, 0x2e0a; 0x2e0d, 0x2e0d; 0x2e1d, 0x2e1d; 0x2e21, 0x2e21] let pi = Cset.of_list [0xab, 0xab; 0x2018, 0x2018; 0x201b, 0x201c; 0x201f, 0x201f; 0x2039, 0x2039; 0x2e02, 0x2e02; 0x2e04, 0x2e04; 0x2e09, 0x2e09; 0x2e0c, 0x2e0c; 0x2e1c, 0x2e1c; 0x2e20, 0x2e20] let po = Cset.of_list [0x21, 0x23; 0x25, 0x27; 0x2a, 0x2a; 0x2c, 0x2c; 0x2e, 0x2f; 0x3a, 0x3b; 0x3f, 0x40; 0x5c, 0x5c; 0xa1, 0xa1; 0xa7, 0xa7; 0xb6, 0xb7; 0xbf, 0xbf; 0x37e, 0x37e; 0x387, 0x387; 0x55a, 0x55f; 0x589, 0x589; 0x5c0, 0x5c0; 0x5c3, 0x5c3; 0x5c6, 0x5c6; 0x5f3, 0x5f4; 0x609, 0x60a; 0x60c, 0x60d; 0x61b, 0x61b; 0x61d, 0x61f; 0x66a, 0x66d; 0x6d4, 0x6d4; 0x700, 0x70d; 0x7f7, 0x7f9; 0x830, 0x83e; 0x85e, 0x85e; 0x964, 0x965; 0x970, 0x970; 0x9fd, 0x9fd; 0xa76, 0xa76; 0xaf0, 0xaf0; 0xc77, 0xc77; 0xc84, 0xc84; 0xdf4, 0xdf4; 0xe4f, 0xe4f; 0xe5a, 0xe5b; 0xf04, 0xf12; 0xf14, 0xf14; 0xf85, 0xf85; 0xfd0, 0xfd4; 0xfd9, 0xfda; 0x104a, 0x104f; 0x10fb, 0x10fb; 0x1360, 0x1368; 0x166e, 0x166e; 0x16eb, 0x16ed; 0x1735, 0x1736; 0x17d4, 0x17d6; 0x17d8, 0x17da; 0x1800, 0x1805; 0x1807, 0x180a; 0x1944, 0x1945; 0x1a1e, 0x1a1f; 0x1aa0, 0x1aa6; 0x1aa8, 0x1aad; 0x1b4e, 0x1b4f; 0x1b5a, 0x1b60; 0x1b7d, 0x1b7f; 0x1bfc, 0x1bff; 0x1c3b, 0x1c3f; 0x1c7e, 0x1c7f; 0x1cc0, 0x1cc7; 0x1cd3, 0x1cd3; 0x2016, 0x2017; 0x2020, 0x2027; 0x2030, 0x2038; 0x203b, 0x203e; 0x2041, 0x2043; 0x2047, 0x2051; 0x2053, 0x2053; 0x2055, 0x205e; 0x2cf9, 0x2cfc; 0x2cfe, 0x2cff; 0x2d70, 0x2d70; 0x2e00, 0x2e01; 0x2e06, 0x2e08; 0x2e0b, 0x2e0b; 0x2e0e, 0x2e16; 0x2e18, 0x2e19; 0x2e1b, 0x2e1b; 0x2e1e, 0x2e1f; 0x2e2a, 0x2e2e; 0x2e30, 0x2e39; 0x2e3c, 0x2e3f; 0x2e41, 0x2e41; 0x2e43, 0x2e4f; 0x2e52, 0x2e54; 0x2e60, 0x2e61; 0x3001, 0x3003; 0x303d, 0x303d; 0x30fb, 0x30fb; 0xa4fe, 0xa4ff; 0xa60d, 0xa60f; 0xa673, 0xa673; 0xa67e, 0xa67e; 0xa6f2, 0xa6f7; 0xa874, 0xa877; 0xa8ce, 0xa8cf; 0xa8f8, 0xa8fa; 0xa8fc, 0xa8fc; 0xa92e, 0xa92f; 0xa95f, 0xa95f; 0xa9c1, 0xa9cd; 0xa9de, 0xa9df; 0xaa5c, 0xaa5f; 0xaade, 0xaadf; 0xaaf0, 0xaaf1; 0xabeb, 0xabeb; 0xfe10, 0xfe16; 0xfe19, 0xfe19; 0xfe30, 0xfe30; 0xfe45, 0xfe46; 0xfe49, 0xfe4c; 0xfe50, 0xfe52; 0xfe54, 0xfe57; 0xfe5f, 0xfe61; 0xfe68, 0xfe68; 0xfe6a, 0xfe6b; 0xff01, 0xff03; 0xff05, 0xff07; 0xff0a, 0xff0a; 0xff0c, 0xff0c; 0xff0e, 0xff0f; 0xff1a, 0xff1b; 0xff1f, 0xff20; 0xff3c, 0xff3c; 0xff61, 0xff61; 0xff64, 0xff65; 0x10100, 0x10102; 0x1039f, 0x1039f; 0x103d0, 0x103d0; 0x1056f, 0x1056f; 0x10857, 0x10857; 0x1091f, 0x1091f; 0x1093f, 0x1093f; 0x10a50, 0x10a58; 0x10a7f, 0x10a7f; 0x10af0, 0x10af6; 0x10b39, 0x10b3f; 0x10b99, 0x10b9c; 0x10ed0, 0x10ed0; 0x10f55, 0x10f59; 0x10f86, 0x10f89; 0x11047, 0x1104d; 0x110bb, 0x110bc; 0x110be, 0x110c1; 0x11140, 0x11143; 0x11174, 0x11175; 0x111c5, 0x111c8; 0x111cd, 0x111cd; 0x111db, 0x111db; 0x111dd, 0x111df; 0x11238, 0x1123d; 0x112a9, 0x112a9; 0x113d4, 0x113d5; 0x113d7, 0x113d8; 0x1144b, 0x1144f; 0x1145a, 0x1145b; 0x1145d, 0x1145d; 0x114c6, 0x114c6; 0x115c1, 0x115d7; 0x11641, 0x11643; 0x11660, 0x1166c; 0x116b9, 0x116b9; 0x1173c, 0x1173e; 0x1183b, 0x1183b; 0x11944, 0x11946; 0x119e2, 0x119e2; 0x11a3f, 0x11a46; 0x11a9a, 0x11a9c; 0x11a9e, 0x11aa2; 0x11b00, 0x11b09; 0x11be1, 0x11be1; 0x11c41, 0x11c45; 0x11c70, 0x11c71; 0x11ef7, 0x11ef8; 0x11f43, 0x11f4f; 0x11fff, 0x11fff; 0x12470, 0x12474; 0x12ff1, 0x12ff2; 0x16a6e, 0x16a6f; 0x16af5, 0x16af5; 0x16b37, 0x16b3b; 0x16b44, 0x16b44; 0x16d6d, 0x16d6f; 0x16e97, 0x16e9a; 0x16fe2, 0x16fe2; 0x1bc9f, 0x1bc9f; 0x1da87, 0x1da8b; 0x1e5ff, 0x1e5ff; 0x1e95e, 0x1e95f] let ps = Cset.of_list [0x28, 0x28; 0x5b, 0x5b; 0x7b, 0x7b; 0xf3a, 0xf3a; 0xf3c, 0xf3c; 0x169b, 0x169b; 0x201a, 0x201a; 0x201e, 0x201e; 0x2045, 0x2045; 0x207d, 0x207d; 0x208d, 0x208d; 0x2308, 0x2308; 0x230a, 0x230a; 0x2329, 0x2329; 0x2768, 0x2768; 0x276a, 0x276a; 0x276c, 0x276c; 0x276e, 0x276e; 0x2770, 0x2770; 0x2772, 0x2772; 0x2774, 0x2774; 0x27c5, 0x27c5; 0x27e6, 0x27e6; 0x27e8, 0x27e8; 0x27ea, 0x27ea; 0x27ec, 0x27ec; 0x27ee, 0x27ee; 0x2983, 0x2983; 0x2985, 0x2985; 0x2987, 0x2987; 0x2989, 0x2989; 0x298b, 0x298b; 0x298d, 0x298d; 0x298f, 0x298f; 0x2991, 0x2991; 0x2993, 0x2993; 0x2995, 0x2995; 0x2997, 0x2997; 0x29d8, 0x29d8; 0x29da, 0x29da; 0x29fc, 0x29fc; 0x2e22, 0x2e22; 0x2e24, 0x2e24; 0x2e26, 0x2e26; 0x2e28, 0x2e28; 0x2e42, 0x2e42; 0x2e55, 0x2e55; 0x2e57, 0x2e57; 0x2e59, 0x2e59; 0x2e5b, 0x2e5b; 0x2e62, 0x2e62; 0x3008, 0x3008; 0x300a, 0x300a; 0x300c, 0x300c; 0x300e, 0x300e; 0x3010, 0x3010; 0x3014, 0x3014; 0x3016, 0x3016; 0x3018, 0x3018; 0x301a, 0x301a; 0x301d, 0x301d; 0xfd3f, 0xfd3f; 0xfe17, 0xfe17; 0xfe35, 0xfe35; 0xfe37, 0xfe37; 0xfe39, 0xfe39; 0xfe3b, 0xfe3b; 0xfe3d, 0xfe3d; 0xfe3f, 0xfe3f; 0xfe41, 0xfe41; 0xfe43, 0xfe43; 0xfe47, 0xfe47; 0xfe59, 0xfe59; 0xfe5b, 0xfe5b; 0xfe5d, 0xfe5d; 0xff08, 0xff08; 0xff3b, 0xff3b; 0xff5b, 0xff5b; 0xff5f, 0xff5f; 0xff62, 0xff62] let sc = Cset.of_list [0x24, 0x24; 0xa2, 0xa5; 0x58f, 0x58f; 0x60b, 0x60b; 0x7fe, 0x7ff; 0x9f2, 0x9f3; 0x9fb, 0x9fb; 0xaf1, 0xaf1; 0xbf9, 0xbf9; 0xe3f, 0xe3f; 0x17db, 0x17db; 0x20a0, 0x20c4; 0xa838, 0xa838; 0xfdfc, 0xfdfc; 0xfe69, 0xfe69; 0xff04, 0xff04; 0xffe0, 0xffe1; 0xffe5, 0xffe6; 0x11fdd, 0x11fe0; 0x1e2ff, 0x1e2ff; 0x1ecb0, 0x1ecb0] let sk = Cset.of_list [0x5e, 0x5e; 0x60, 0x60; 0xa8, 0xa8; 0xaf, 0xaf; 0xb4, 0xb4; 0xb8, 0xb8; 0x2c2, 0x2c5; 0x2d2, 0x2df; 0x2e5, 0x2eb; 0x2ed, 0x2ed; 0x2ef, 0x2ff; 0x375, 0x375; 0x384, 0x385; 0x888, 0x888; 0x1fbd, 0x1fbd; 0x1fbf, 0x1fc1; 0x1fcd, 0x1fcf; 0x1fdd, 0x1fdf; 0x1fed, 0x1fef; 0x1ffd, 0x1ffe; 0x309b, 0x309c; 0xa700, 0xa716; 0xa720, 0xa721; 0xa789, 0xa78a; 0xab5b, 0xab5b; 0xab6a, 0xab6b; 0xfbb2, 0xfbc2; 0xff3e, 0xff3e; 0xff40, 0xff40; 0xffe3, 0xffe3; 0x10ec9, 0x10eca; 0x1f3fb, 0x1f3ff] let sm = Cset.of_list [0x2b, 0x2b; 0x3c, 0x3e; 0x7c, 0x7c; 0x7e, 0x7e; 0xac, 0xac; 0xb1, 0xb1; 0xd7, 0xd7; 0xf7, 0xf7; 0x3f6, 0x3f6; 0x606, 0x608; 0x2044, 0x2044; 0x2052, 0x2052; 0x207a, 0x207c; 0x208a, 0x208c; 0x2118, 0x2118; 0x2140, 0x2144; 0x214b, 0x214b; 0x2190, 0x2194; 0x219a, 0x219b; 0x21a0, 0x21a0; 0x21a3, 0x21a3; 0x21a6, 0x21a6; 0x21ae, 0x21ae; 0x21ce, 0x21cf; 0x21d2, 0x21d2; 0x21d4, 0x21d4; 0x21f4, 0x22ff; 0x2320, 0x2321; 0x237c, 0x237c; 0x239b, 0x23b3; 0x23dc, 0x23e1; 0x25b7, 0x25b7; 0x25c1, 0x25c1; 0x25f8, 0x25ff; 0x266f, 0x266f; 0x27c0, 0x27c4; 0x27c7, 0x27e5; 0x27f0, 0x27ff; 0x2900, 0x2982; 0x2999, 0x29d7; 0x29dc, 0x29fb; 0x29fe, 0x2aff; 0x2b30, 0x2b44; 0x2b47, 0x2b4c; 0xfb29, 0xfb29; 0xfe62, 0xfe62; 0xfe64, 0xfe66; 0xff0b, 0xff0b; 0xff1c, 0xff1e; 0xff5c, 0xff5c; 0xff5e, 0xff5e; 0xffe2, 0xffe2; 0xffe9, 0xffec; 0x10d8e, 0x10d8f; 0x1cedd, 0x1cedf; 0x1cef0, 0x1cefd; 0x1d6c1, 0x1d6c1; 0x1d6db, 0x1d6db; 0x1d6fb, 0x1d6fb; 0x1d715, 0x1d715; 0x1d735, 0x1d735; 0x1d74f, 0x1d74f; 0x1d76f, 0x1d76f; 0x1d789, 0x1d789; 0x1d7a9, 0x1d7a9; 0x1d7c3, 0x1d7c3; 0x1db00, 0x1db1c; 0x1eef0, 0x1eef1; 0x1f8d0, 0x1f8d8] let so = Cset.of_list [0xa6, 0xa6; 0xa9, 0xa9; 0xae, 0xae; 0xb0, 0xb0; 0x482, 0x482; 0x58d, 0x58e; 0x60e, 0x60f; 0x6de, 0x6de; 0x6e9, 0x6e9; 0x6fd, 0x6fe; 0x7f6, 0x7f6; 0x9fa, 0x9fa; 0xb70, 0xb70; 0xbf3, 0xbf8; 0xbfa, 0xbfa; 0xc7f, 0xc7f; 0xd4f, 0xd4f; 0xd79, 0xd79; 0xf01, 0xf03; 0xf13, 0xf13; 0xf15, 0xf17; 0xf1a, 0xf1f; 0xf34, 0xf34; 0xf36, 0xf36; 0xf38, 0xf38; 0xfbe, 0xfc5; 0xfc7, 0xfcc; 0xfce, 0xfcf; 0xfd5, 0xfd8; 0x109e, 0x109f; 0x1390, 0x1399; 0x166d, 0x166d; 0x1940, 0x1940; 0x19de, 0x19ff; 0x1b61, 0x1b6a; 0x1b74, 0x1b7c; 0x2100, 0x2101; 0x2103, 0x2106; 0x2108, 0x2109; 0x2114, 0x2114; 0x2116, 0x2117; 0x211e, 0x2123; 0x2125, 0x2125; 0x2127, 0x2127; 0x2129, 0x2129; 0x212e, 0x212e; 0x213a, 0x213b; 0x214a, 0x214a; 0x214c, 0x214d; 0x214f, 0x214f; 0x218a, 0x218b; 0x2195, 0x2199; 0x219c, 0x219f; 0x21a1, 0x21a2; 0x21a4, 0x21a5; 0x21a7, 0x21ad; 0x21af, 0x21cd; 0x21d0, 0x21d1; 0x21d3, 0x21d3; 0x21d5, 0x21f3; 0x2300, 0x2307; 0x230c, 0x231f; 0x2322, 0x2328; 0x232b, 0x237b; 0x237d, 0x239a; 0x23b4, 0x23db; 0x23e2, 0x2429; 0x2440, 0x244a; 0x249c, 0x24e9; 0x2500, 0x25b6; 0x25b8, 0x25c0; 0x25c2, 0x25f7; 0x2600, 0x266e; 0x2670, 0x2767; 0x2794, 0x27bf; 0x2800, 0x28ff; 0x2b00, 0x2b2f; 0x2b45, 0x2b46; 0x2b4d, 0x2b73; 0x2b76, 0x2bff; 0x2ce5, 0x2cea; 0x2e50, 0x2e51; 0x2e80, 0x2e99; 0x2e9b, 0x2ef3; 0x2f00, 0x2fd5; 0x2ff0, 0x2fff; 0x3004, 0x3004; 0x3012, 0x3013; 0x3020, 0x3020; 0x3036, 0x3037; 0x303e, 0x303f; 0x3190, 0x3191; 0x3196, 0x319f; 0x31c0, 0x31e5; 0x31ef, 0x31ef; 0x3200, 0x321e; 0x322a, 0x3247; 0x3250, 0x3250; 0x3260, 0x327f; 0x328a, 0x32b0; 0x32c0, 0x33ff; 0x4dc0, 0x4dff; 0xa490, 0xa4c6; 0xa828, 0xa82b; 0xa836, 0xa837; 0xa839, 0xa839; 0xaa77, 0xaa79; 0xfbc3, 0xfbd2; 0xfd40, 0xfd4f; 0xfd90, 0xfd91; 0xfdc8, 0xfdcf; 0xfdfd, 0xfdff; 0xffe4, 0xffe4; 0xffe8, 0xffe8; 0xffed, 0xffee; 0xfffc, 0xfffd; 0x10137, 0x1013f; 0x10179, 0x10189; 0x1018c, 0x1018e; 0x10190, 0x1019c; 0x101a0, 0x101a0; 0x101d0, 0x101fc; 0x10877, 0x10878; 0x10ac8, 0x10ac8; 0x10ed1, 0x10ed8; 0x1173f, 0x1173f; 0x11fd5, 0x11fdc; 0x11fe1, 0x11ff1; 0x16b3c, 0x16b3f; 0x16b45, 0x16b45; 0x1bc9c, 0x1bc9c; 0x1cc00, 0x1ccef; 0x1ccfa, 0x1ccfc; 0x1cd00, 0x1ceb3; 0x1ceba, 0x1ced0; 0x1ced2, 0x1ced4; 0x1cee0, 0x1ceef; 0x1cf50, 0x1cfc3; 0x1d000, 0x1d0f5; 0x1d100, 0x1d126; 0x1d129, 0x1d164; 0x1d16a, 0x1d16c; 0x1d183, 0x1d184; 0x1d18c, 0x1d1a9; 0x1d1ae, 0x1d241; 0x1d245, 0x1d245; 0x1d253, 0x1d25a; 0x1d25d, 0x1d25e; 0x1d260, 0x1d27f; 0x1d300, 0x1d356; 0x1d800, 0x1d9ff; 0x1da37, 0x1da3a; 0x1da6d, 0x1da74; 0x1da76, 0x1da83; 0x1da85, 0x1da86; 0x1e14f, 0x1e14f; 0x1ecac, 0x1ecac; 0x1ed2e, 0x1ed2e; 0x1f000, 0x1f02b; 0x1f030, 0x1f093; 0x1f0a0, 0x1f0ae; 0x1f0b1, 0x1f0bf; 0x1f0c1, 0x1f0cf; 0x1f0d1, 0x1f0f5; 0x1f10d, 0x1f1ae; 0x1f1e6, 0x1f202; 0x1f210, 0x1f23b; 0x1f240, 0x1f248; 0x1f250, 0x1f251; 0x1f260, 0x1f265; 0x1f300, 0x1f3fa; 0x1f400, 0x1f6d9; 0x1f6dc, 0x1f6ec; 0x1f6f0, 0x1f6fc; 0x1f700, 0x1f7db; 0x1f7e0, 0x1f7eb; 0x1f7f0, 0x1f80b; 0x1f810, 0x1f847; 0x1f850, 0x1f859; 0x1f860, 0x1f887; 0x1f890, 0x1f8ad; 0x1f8b0, 0x1f8bb; 0x1f8c0, 0x1f8c1; 0x1f900, 0x1fa57; 0x1fa60, 0x1fa6d; 0x1fa70, 0x1fa7c; 0x1fa80, 0x1fac6; 0x1fac8, 0x1fac8; 0x1facc, 0x1fadd; 0x1fadf, 0x1faeb; 0x1faef, 0x1fafa; 0x1fb00, 0x1fb92; 0x1fb94, 0x1fbef; 0x1fbfa, 0x1fbfa] let zl = Cset.of_list [0x2028, 0x2028] let zp = Cset.of_list [0x2029, 0x2029] let zs = Cset.of_list [0x20, 0x20; 0xa0, 0xa0; 0x1680, 0x1680; 0x2000, 0x200a; 0x202f, 0x202f; 0x205f, 0x205f; 0x3000, 0x3000] let list = [ ("cc", cc); ("cf", cf); ("cn", cn); ("co", co); ("cs", cs); ("ll", ll); ("lm", lm); ("lo", lo); ("lt", lt); ("lu", lu); ("mc", mc); ("me", me); ("mn", mn); ("nd", nd); ("nl", nl); ("no", no); ("pc", pc); ("pd", pd); ("pe", pe); ("pf", pf); ("pi", pi); ("po", po); ("ps", ps); ("sc", sc); ("sk", sk); ("sm", sm); ("so", so); ("zl", zl); ("zp", zp); ("zs", zs) ] end module Properties = struct let alphabetic = Cset.of_list [0x41, 0x5a; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x345, 0x345; 0x363, 0x374; 0x376, 0x377; 0x37a, 0x37d; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x48a, 0x52f; 0x531, 0x556; 0x558, 0x559; 0x560, 0x588; 0x58b, 0x58c; 0x5b0, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c9; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x610, 0x61a; 0x620, 0x657; 0x659, 0x65f; 0x66e, 0x6d3; 0x6d5, 0x6dc; 0x6e1, 0x6e8; 0x6ed, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x73f; 0x74d, 0x7b1; 0x7ca, 0x7ea; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x800, 0x817; 0x81a, 0x82c; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x897, 0x897; 0x8a0, 0x8c9; 0x8d4, 0x8df; 0x8e3, 0x8e9; 0x8f0, 0x93b; 0x93d, 0x94c; 0x94e, 0x950; 0x955, 0x963; 0x971, 0x983; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9cc; 0x9ce, 0x9ce; 0x9d7, 0x9d7; 0x9dc, 0x9dd; 0x9df, 0x9e3; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa01, 0xa03; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4c; 0xa51, 0xa51; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa70, 0xa75; 0xa81, 0xa83; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xac5; 0xac7, 0xac9; 0xacb, 0xacc; 0xad0, 0xad0; 0xae0, 0xae3; 0xaf9, 0xafc; 0xb01, 0xb03; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4c; 0xb56, 0xb57; 0xb5c, 0xb5d; 0xb5f, 0xb63; 0xb71, 0xb71; 0xb82, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcc; 0xbd0, 0xbd0; 0xbd7, 0xbd7; 0xc00, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4c; 0xc55, 0xc56; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc63; 0xc80, 0xc83; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccc; 0xcd5, 0xcd6; 0xcdc, 0xcde; 0xce0, 0xce3; 0xcf1, 0xcf3; 0xd00, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4c; 0xd4e, 0xd4e; 0xd54, 0xd57; 0xd5f, 0xd63; 0xd7a, 0xd7f; 0xd81, 0xd83; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xdf2, 0xdf3; 0xe01, 0xe3a; 0xe40, 0xe46; 0xe4d, 0xe4d; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb9; 0xebb, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xecd, 0xecd; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf71, 0xf83; 0xf88, 0xf97; 0xf99, 0xfbc; 0x1000, 0x1036; 0x1038, 0x1038; 0x103b, 0x103f; 0x1050, 0x108f; 0x109a, 0x109d; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1713; 0x171f, 0x1733; 0x1740, 0x1753; 0x1760, 0x176c; 0x176e, 0x1770; 0x1772, 0x1773; 0x1780, 0x17b3; 0x17b6, 0x17c8; 0x17d7, 0x17d7; 0x17dc, 0x17dc; 0x1820, 0x1878; 0x1880, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1920, 0x192b; 0x1930, 0x1938; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a1b; 0x1a20, 0x1a5e; 0x1a61, 0x1a74; 0x1aa7, 0x1aa7; 0x1abf, 0x1ac0; 0x1acc, 0x1ace; 0x1b00, 0x1b33; 0x1b35, 0x1b43; 0x1b45, 0x1b4c; 0x1b80, 0x1ba9; 0x1bac, 0x1baf; 0x1bba, 0x1be5; 0x1be7, 0x1bf1; 0x1c00, 0x1c36; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x1d00, 0x1dbf; 0x1dd3, 0x1df4; 0x1e00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x2071, 0x2071; 0x207f, 0x207f; 0x208f, 0x209f; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x212d; 0x212f, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x24b6, 0x24e9; 0x2c00, 0x2ce4; 0x2ceb, 0x2cee; 0x2cf2, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x2de0, 0x2dff; 0x2e2f, 0x2e2f; 0x3005, 0x3007; 0x3021, 0x3029; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x309d, 0x309f; 0x30a1, 0x30fa; 0x30fc, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa640, 0xa66e; 0xa674, 0xa67b; 0xa67f, 0xa6ef; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f1, 0xa805; 0xa807, 0xa827; 0xa840, 0xa873; 0xa880, 0xa8c3; 0xa8c5, 0xa8c5; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8ff; 0xa90a, 0xa92a; 0xa930, 0xa952; 0xa960, 0xa97c; 0xa980, 0xa9b2; 0xa9b4, 0xa9bf; 0xa9cf, 0xa9cf; 0xa9e0, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa36; 0xaa40, 0xaa4d; 0xaa60, 0xaa76; 0xaa7a, 0xaabe; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaef; 0xaaf2, 0xaaf5; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab6c, 0xab6d; 0xab70, 0xabea; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff21, 0xff3a; 0xff41, 0xff5a; 0xff66, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x1037a; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d27; 0x10d4a, 0x10d65; 0x10d69, 0x10d69; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eab, 0x10eac; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10ecb, 0x10ecd; 0x10ed9, 0x10eee; 0x10ef3, 0x10ef3; 0x10ef5, 0x10ef5; 0x10ef7, 0x10ef8; 0x10efa, 0x10efc; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11000, 0x11045; 0x11071, 0x11075; 0x11080, 0x110b8; 0x110c2, 0x110c2; 0x110d0, 0x110e8; 0x11100, 0x11132; 0x11144, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11180, 0x111bf; 0x111c1, 0x111c4; 0x111ce, 0x111cf; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x11234; 0x11237, 0x11237; 0x1123e, 0x11241; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112e8; 0x11300, 0x11303; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134c; 0x11350, 0x11350; 0x11357, 0x11357; 0x1135d, 0x11363; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113cd; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11441; 0x11443, 0x11445; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114c1; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115b5; 0x115b8, 0x115be; 0x115d8, 0x115dd; 0x11600, 0x1163e; 0x11640, 0x11640; 0x11644, 0x11644; 0x11680, 0x116b5; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x1171d, 0x1172a; 0x11740, 0x11746; 0x11800, 0x11838; 0x118a0, 0x118df; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x11935; 0x11937, 0x11938; 0x1193b, 0x1193c; 0x1193f, 0x11942; 0x119a0, 0x119a7; 0x119aa, 0x119d7; 0x119da, 0x119df; 0x119e1, 0x119e1; 0x119e3, 0x119e4; 0x11a00, 0x11a32; 0x11a35, 0x11a3e; 0x11a50, 0x11a97; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b0a, 0x11b0a; 0x11b60, 0x11b67; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c36; 0x11c38, 0x11c3e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d41; 0x11d43, 0x11d43; 0x11d46, 0x11d47; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d96; 0x11d98, 0x11d98; 0x11db0, 0x11ddb; 0x11df0, 0x11df1; 0x11ee0, 0x11ef6; 0x11f00, 0x11f10; 0x11f12, 0x11f3a; 0x11f3e, 0x11f40; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246f; 0x12475, 0x12543; 0x12550, 0x12686; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1612e; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b40, 0x16b43; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f4f, 0x16f87; 0x16f8f, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff0, 0x16ff6; 0x17000, 0x18cda; 0x18cff, 0x18d20; 0x18d80, 0x18df2; 0x18e00, 0x19191; 0x191a0, 0x191d2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b128; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b168; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1bc9e, 0x1bc9e; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1df00, 0x1df81; 0x1df90, 0x1df96; 0x1dfcd, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e030, 0x1e06d; 0x1e08f, 0x1e08f; 0x1e100, 0x1e12c; 0x1e137, 0x1e13d; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4eb; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6f5; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e900, 0x1e943; 0x1e947, 0x1e947; 0x1e94b, 0x1e94b; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189; 0x20000, 0x2a6df; 0x2a700, 0x2b81e; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0x3d000, 0x3fc3f] let ascii_hex_digit = Cset.of_list [0x30, 0x39; 0x41, 0x46; 0x61, 0x66] let hex_digit = Cset.of_list [0x30, 0x39; 0x41, 0x46; 0x61, 0x66; 0xff10, 0xff19; 0xff21, 0xff26; 0xff41, 0xff46] let id_continue = Cset.of_list [0x30, 0x39; 0x41, 0x5a; 0x5f, 0x5f; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xb7, 0xb7; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x300, 0x374; 0x376, 0x377; 0x37a, 0x37d; 0x37f, 0x37f; 0x386, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x483, 0x487; 0x48a, 0x52f; 0x531, 0x556; 0x558, 0x559; 0x560, 0x588; 0x58b, 0x58c; 0x591, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c9; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x610, 0x61a; 0x620, 0x669; 0x66e, 0x6d3; 0x6d5, 0x6dc; 0x6df, 0x6e8; 0x6ea, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x74a; 0x74d, 0x7b1; 0x7c0, 0x7f5; 0x7fa, 0x7fa; 0x7fd, 0x7fd; 0x800, 0x82d; 0x840, 0x85b; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x897, 0x8e1; 0x8e3, 0x963; 0x966, 0x96f; 0x971, 0x983; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bc, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9ce; 0x9d7, 0x9d7; 0x9dc, 0x9dd; 0x9df, 0x9e3; 0x9e6, 0x9f1; 0x9fc, 0x9fc; 0x9fe, 0x9fe; 0xa01, 0xa03; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa3c, 0xa3c; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4d; 0xa51, 0xa51; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa66, 0xa75; 0xa81, 0xa83; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabc, 0xac5; 0xac7, 0xac9; 0xacb, 0xacd; 0xad0, 0xad0; 0xae0, 0xae3; 0xae6, 0xaef; 0xaf9, 0xaff; 0xb01, 0xb03; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3c, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4d; 0xb53, 0xb57; 0xb5c, 0xb5d; 0xb5f, 0xb63; 0xb66, 0xb6f; 0xb71, 0xb71; 0xb82, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcd; 0xbd0, 0xbd0; 0xbd7, 0xbd7; 0xbe6, 0xbef; 0xc00, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3c, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4d; 0xc55, 0xc56; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc63; 0xc66, 0xc6f; 0xc80, 0xc83; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbc, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccd; 0xcd5, 0xcd6; 0xcdc, 0xcde; 0xce0, 0xce3; 0xce6, 0xcef; 0xcf1, 0xcf3; 0xd00, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4e; 0xd54, 0xd57; 0xd5f, 0xd63; 0xd66, 0xd6f; 0xd7a, 0xd7f; 0xd81, 0xd83; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xdca, 0xdca; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xde6, 0xdef; 0xdf2, 0xdf3; 0xe01, 0xe3a; 0xe40, 0xe4e; 0xe50, 0xe59; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xec8, 0xece; 0xed0, 0xed9; 0xedc, 0xedf; 0xf00, 0xf00; 0xf18, 0xf19; 0xf20, 0xf29; 0xf35, 0xf35; 0xf37, 0xf37; 0xf39, 0xf39; 0xf3e, 0xf47; 0xf49, 0xf6c; 0xf71, 0xf84; 0xf86, 0xf97; 0xf99, 0xfbc; 0xfc6, 0xfc6; 0x1000, 0x1049; 0x1050, 0x109d; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x135d, 0x135f; 0x1369, 0x1371; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1715; 0x171f, 0x1734; 0x1740, 0x1753; 0x1760, 0x176c; 0x176e, 0x1770; 0x1772, 0x1773; 0x1780, 0x17d3; 0x17d7, 0x17d7; 0x17dc, 0x17dd; 0x17e0, 0x17e9; 0x180b, 0x180d; 0x180f, 0x1819; 0x1820, 0x1878; 0x1880, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1920, 0x192b; 0x1930, 0x193b; 0x1946, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x19d0, 0x19da; 0x1a00, 0x1a1b; 0x1a20, 0x1a5e; 0x1a60, 0x1a7c; 0x1a7f, 0x1a89; 0x1a90, 0x1a99; 0x1aa7, 0x1aa7; 0x1ab0, 0x1abd; 0x1abf, 0x1af0; 0x1b00, 0x1b4c; 0x1b50, 0x1b59; 0x1b6b, 0x1b73; 0x1b80, 0x1bf3; 0x1c00, 0x1c37; 0x1c40, 0x1c49; 0x1c4d, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1cd0, 0x1cd2; 0x1cd4, 0x1cfa; 0x1d00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x200c, 0x200d; 0x203f, 0x2040; 0x2054, 0x2054; 0x2071, 0x2071; 0x207f, 0x207f; 0x208f, 0x209f; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20f0; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d7f, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x2de0, 0x2dff; 0x3005, 0x3007; 0x3021, 0x302f; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x3099, 0x309f; 0x30a1, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa62b; 0xa640, 0xa66f; 0xa674, 0xa67d; 0xa67f, 0xa6f1; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f1, 0xa827; 0xa82c, 0xa82c; 0xa840, 0xa873; 0xa880, 0xa8c5; 0xa8d0, 0xa8d9; 0xa8e0, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa92d; 0xa930, 0xa953; 0xa960, 0xa97c; 0xa980, 0xa9c0; 0xa9cf, 0xa9d9; 0xa9e0, 0xa9fe; 0xaa00, 0xaa36; 0xaa40, 0xaa4d; 0xaa50, 0xaa59; 0xaa60, 0xaa76; 0xaa7a, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaef; 0xaaf2, 0xaaf6; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab6c, 0xab6d; 0xab70, 0xabea; 0xabec, 0xabed; 0xabf0, 0xabf9; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe00, 0xfe0f; 0xfe20, 0xfe2f; 0xfe33, 0xfe34; 0xfe4d, 0xfe4f; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff10, 0xff19; 0xff21, 0xff3a; 0xff3f, 0xff3f; 0xff41, 0xff5a; 0xff65, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x101fd, 0x101fd; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x102e0, 0x102e0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x1037a; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104a0, 0x104a9; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a38, 0x10a3a; 0x10a3f, 0x10a3f; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae6; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d27; 0x10d30, 0x10d39; 0x10d40, 0x10d65; 0x10d69, 0x10d6d; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eab, 0x10eac; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10ecb, 0x10ecf; 0x10ed9, 0x10eee; 0x10ef0, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f50; 0x10f70, 0x10f85; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11000, 0x11046; 0x11066, 0x11075; 0x1107f, 0x110ba; 0x110c2, 0x110c2; 0x110d0, 0x110e8; 0x110f0, 0x110f9; 0x11100, 0x11134; 0x11136, 0x1113f; 0x11144, 0x11147; 0x11150, 0x11173; 0x11176, 0x11176; 0x11180, 0x111c4; 0x111c9, 0x111cc; 0x111ce, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x11237; 0x1123e, 0x11241; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112ea; 0x112f0, 0x112f9; 0x11300, 0x11303; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133b, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134d; 0x11350, 0x11350; 0x11357, 0x11357; 0x1135d, 0x11363; 0x11366, 0x1136c; 0x11370, 0x11374; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113d3; 0x113e1, 0x113e2; 0x11400, 0x1144a; 0x11450, 0x11459; 0x1145e, 0x11461; 0x11480, 0x114c5; 0x114c7, 0x114c7; 0x114d0, 0x114d9; 0x11580, 0x115b5; 0x115b8, 0x115c0; 0x115d8, 0x115dd; 0x11600, 0x11640; 0x11644, 0x11644; 0x11650, 0x11659; 0x11680, 0x116b8; 0x116c0, 0x116c9; 0x116d0, 0x116e3; 0x11700, 0x1171a; 0x1171d, 0x1172b; 0x11730, 0x11739; 0x11740, 0x11746; 0x11800, 0x1183a; 0x118a0, 0x118e9; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x11935; 0x11937, 0x11938; 0x1193b, 0x11943; 0x11950, 0x11959; 0x119a0, 0x119a7; 0x119aa, 0x119d7; 0x119da, 0x119e1; 0x119e3, 0x119e4; 0x11a00, 0x11a3e; 0x11a47, 0x11a47; 0x11a50, 0x11a99; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b0a, 0x11b0a; 0x11b60, 0x11b67; 0x11bc0, 0x11be0; 0x11bf0, 0x11bf9; 0x11c00, 0x11c08; 0x11c0a, 0x11c36; 0x11c38, 0x11c40; 0x11c50, 0x11c59; 0x11c72, 0x11c8f; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d47; 0x11d50, 0x11d59; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d98; 0x11da0, 0x11da9; 0x11db0, 0x11ddb; 0x11de0, 0x11de9; 0x11df0, 0x11df1; 0x11ee0, 0x11ef6; 0x11f00, 0x11f10; 0x11f12, 0x11f3a; 0x11f3e, 0x11f42; 0x11f50, 0x11f5a; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246f; 0x12475, 0x12543; 0x12550, 0x12686; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13440, 0x13455; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x16139; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a60, 0x16a69; 0x16a70, 0x16abe; 0x16ac0, 0x16ac9; 0x16ad0, 0x16aed; 0x16af0, 0x16af4; 0x16b00, 0x16b36; 0x16b40, 0x16b43; 0x16b50, 0x16b59; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16d70, 0x16d79; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f4f, 0x16f87; 0x16f8f, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe4; 0x16ff0, 0x16ff6; 0x17000, 0x18cda; 0x18cff, 0x18d20; 0x18d80, 0x18df2; 0x18e00, 0x19191; 0x191a0, 0x191d2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b128; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b168; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1bc9d, 0x1bc9e; 0x1ccf0, 0x1ccf9; 0x1cf00, 0x1cf2d; 0x1cf30, 0x1cf46; 0x1d127, 0x1d128; 0x1d165, 0x1d169; 0x1d16d, 0x1d172; 0x1d17b, 0x1d182; 0x1d185, 0x1d18b; 0x1d1aa, 0x1d1ad; 0x1d242, 0x1d244; 0x1d250, 0x1d252; 0x1d25b, 0x1d25c; 0x1d25f, 0x1d25f; 0x1d280, 0x1d281; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1da00, 0x1da36; 0x1da3b, 0x1da6c; 0x1da75, 0x1da75; 0x1da84, 0x1da84; 0x1da9b, 0x1da9f; 0x1daa1, 0x1daaf; 0x1df00, 0x1df81; 0x1df90, 0x1df96; 0x1dfcd, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e030, 0x1e06d; 0x1e08f, 0x1e08f; 0x1e100, 0x1e12c; 0x1e130, 0x1e13d; 0x1e140, 0x1e149; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ae; 0x1e2c0, 0x1e2f9; 0x1e4d0, 0x1e4f9; 0x1e5d0, 0x1e5fa; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6f5; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e8d0, 0x1e8d6; 0x1e900, 0x1e94b; 0x1e950, 0x1e959; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1fbf0, 0x1fbf9; 0x20000, 0x2a6df; 0x2a700, 0x2b81e; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0x3d000, 0x3fc3f; 0xe0100, 0xe01ef] let id_start = Cset.of_list [0x41, 0x5a; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x370, 0x374; 0x376, 0x377; 0x37a, 0x37d; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x48a, 0x52f; 0x531, 0x556; 0x558, 0x559; 0x560, 0x588; 0x58b, 0x58c; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x620, 0x64a; 0x66e, 0x66f; 0x671, 0x6d3; 0x6d5, 0x6d5; 0x6e5, 0x6e6; 0x6ee, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x710; 0x712, 0x72f; 0x74d, 0x7a5; 0x7b1, 0x7b1; 0x7ca, 0x7ea; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x800, 0x815; 0x81a, 0x81a; 0x824, 0x824; 0x828, 0x828; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x8a0, 0x8c9; 0x904, 0x939; 0x93d, 0x93d; 0x950, 0x950; 0x958, 0x961; 0x971, 0x980; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9bd; 0x9ce, 0x9ce; 0x9dc, 0x9dd; 0x9df, 0x9e1; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa72, 0xa74; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xabd; 0xad0, 0xad0; 0xae0, 0xae1; 0xaf9, 0xaf9; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb3d; 0xb5c, 0xb5d; 0xb5f, 0xb61; 0xb71, 0xb71; 0xb83, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbd0, 0xbd0; 0xc05, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc3d; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc61; 0xc80, 0xc80; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcbd; 0xcdc, 0xcde; 0xce0, 0xce1; 0xcf1, 0xcf2; 0xd04, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd3d; 0xd4e, 0xd4e; 0xd54, 0xd56; 0xd5f, 0xd61; 0xd7a, 0xd7f; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xe01, 0xe30; 0xe32, 0xe33; 0xe40, 0xe46; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb0; 0xeb2, 0xeb3; 0xebd, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf88, 0xf8c; 0x1000, 0x102a; 0x103f, 0x103f; 0x1050, 0x1055; 0x105a, 0x105d; 0x1061, 0x1061; 0x1065, 0x1066; 0x106e, 0x1070; 0x1075, 0x1081; 0x108e, 0x108e; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1711; 0x171f, 0x1731; 0x1740, 0x1751; 0x1760, 0x176c; 0x176e, 0x1770; 0x1780, 0x17b3; 0x17d7, 0x17d7; 0x17dc, 0x17dc; 0x1820, 0x1878; 0x1880, 0x18a8; 0x18aa, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a16; 0x1a20, 0x1a54; 0x1aa7, 0x1aa7; 0x1b05, 0x1b33; 0x1b45, 0x1b4c; 0x1b83, 0x1ba0; 0x1bae, 0x1baf; 0x1bba, 0x1be5; 0x1c00, 0x1c23; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x1d00, 0x1dbf; 0x1e00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x2071, 0x2071; 0x207f, 0x207f; 0x208f, 0x209f; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cee; 0x2cf2, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x3005, 0x3007; 0x3021, 0x3029; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x309b, 0x309f; 0x30a1, 0x30fa; 0x30fc, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa640, 0xa66e; 0xa67f, 0xa69d; 0xa6a0, 0xa6ef; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f1, 0xa801; 0xa803, 0xa805; 0xa807, 0xa80a; 0xa80c, 0xa822; 0xa840, 0xa873; 0xa882, 0xa8b3; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8fe; 0xa90a, 0xa925; 0xa930, 0xa946; 0xa960, 0xa97c; 0xa984, 0xa9b2; 0xa9cf, 0xa9cf; 0xa9e0, 0xa9e4; 0xa9e6, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa28; 0xaa40, 0xaa42; 0xaa44, 0xaa4b; 0xaa60, 0xaa76; 0xaa7a, 0xaa7a; 0xaa7e, 0xaaaf; 0xaab1, 0xaab1; 0xaab5, 0xaab6; 0xaab9, 0xaabd; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaea; 0xaaf2, 0xaaf4; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab6c, 0xab6d; 0xab70, 0xabe2; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb1d; 0xfb1f, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdfb; 0xfe70, 0xfe74; 0xfe76, 0xfefc; 0xff21, 0xff3a; 0xff41, 0xff5a; 0xff66, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x10375; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a00; 0x10a10, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d23; 0x10d4a, 0x10d65; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10ed9, 0x10eee; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11003, 0x11037; 0x11071, 0x11072; 0x11075, 0x11075; 0x11083, 0x110af; 0x110d0, 0x110e8; 0x11103, 0x11126; 0x11144, 0x11144; 0x11147, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11183, 0x111b2; 0x111c1, 0x111c4; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x1122b; 0x1123f, 0x11240; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112de; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x1133d; 0x11350, 0x11350; 0x1135d, 0x11361; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113b7; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11434; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114af; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115ae; 0x115d8, 0x115db; 0x11600, 0x1162f; 0x11644, 0x11644; 0x11680, 0x116aa; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x11740, 0x11746; 0x11800, 0x1182b; 0x118a0, 0x118df; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x1192f; 0x1193f, 0x1193f; 0x11941, 0x11941; 0x119a0, 0x119a7; 0x119aa, 0x119d0; 0x119e1, 0x119e1; 0x119e3, 0x119e3; 0x11a00, 0x11a00; 0x11a0b, 0x11a32; 0x11a3a, 0x11a3a; 0x11a50, 0x11a50; 0x11a5c, 0x11a89; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b0a, 0x11b0a; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c2e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d30; 0x11d46, 0x11d46; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d89; 0x11d98, 0x11d98; 0x11db0, 0x11ddb; 0x11df1, 0x11df1; 0x11ee0, 0x11ef2; 0x11f02, 0x11f02; 0x11f04, 0x11f10; 0x11f12, 0x11f33; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246f; 0x12475, 0x12543; 0x12550, 0x12686; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1611d; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b40, 0x16b43; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f50, 0x16f50; 0x16f93, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff2, 0x16ff6; 0x17000, 0x18cda; 0x18cff, 0x18d20; 0x18d80, 0x18df2; 0x18e00, 0x19191; 0x191a0, 0x191d2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b128; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b168; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1df00, 0x1df81; 0x1df90, 0x1df96; 0x1dfcd, 0x1dfff; 0x1e030, 0x1e06d; 0x1e100, 0x1e12c; 0x1e137, 0x1e13d; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4eb; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6e2; 0x1e6e4, 0x1e6e5; 0x1e6e7, 0x1e6ed; 0x1e6f0, 0x1e6f4; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e900, 0x1e943; 0x1e94b, 0x1e94b; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x20000, 0x2a6df; 0x2a700, 0x2b81e; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0x3d000, 0x3fc3f] let lowercase = Cset.of_list [0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xdf, 0xf6; 0xf8, 0xff; 0x101, 0x101; 0x103, 0x103; 0x105, 0x105; 0x107, 0x107; 0x109, 0x109; 0x10b, 0x10b; 0x10d, 0x10d; 0x10f, 0x10f; 0x111, 0x111; 0x113, 0x113; 0x115, 0x115; 0x117, 0x117; 0x119, 0x119; 0x11b, 0x11b; 0x11d, 0x11d; 0x11f, 0x11f; 0x121, 0x121; 0x123, 0x123; 0x125, 0x125; 0x127, 0x127; 0x129, 0x129; 0x12b, 0x12b; 0x12d, 0x12d; 0x12f, 0x12f; 0x131, 0x131; 0x133, 0x133; 0x135, 0x135; 0x137, 0x138; 0x13a, 0x13a; 0x13c, 0x13c; 0x13e, 0x13e; 0x140, 0x140; 0x142, 0x142; 0x144, 0x144; 0x146, 0x146; 0x148, 0x149; 0x14b, 0x14b; 0x14d, 0x14d; 0x14f, 0x14f; 0x151, 0x151; 0x153, 0x153; 0x155, 0x155; 0x157, 0x157; 0x159, 0x159; 0x15b, 0x15b; 0x15d, 0x15d; 0x15f, 0x15f; 0x161, 0x161; 0x163, 0x163; 0x165, 0x165; 0x167, 0x167; 0x169, 0x169; 0x16b, 0x16b; 0x16d, 0x16d; 0x16f, 0x16f; 0x171, 0x171; 0x173, 0x173; 0x175, 0x175; 0x177, 0x177; 0x17a, 0x17a; 0x17c, 0x17c; 0x17e, 0x180; 0x183, 0x183; 0x185, 0x185; 0x188, 0x188; 0x18c, 0x18d; 0x192, 0x192; 0x195, 0x195; 0x199, 0x19b; 0x19e, 0x19e; 0x1a1, 0x1a1; 0x1a3, 0x1a3; 0x1a5, 0x1a5; 0x1a8, 0x1a8; 0x1aa, 0x1ab; 0x1ad, 0x1ad; 0x1b0, 0x1b0; 0x1b4, 0x1b4; 0x1b6, 0x1b6; 0x1b9, 0x1ba; 0x1bd, 0x1bf; 0x1c6, 0x1c6; 0x1c9, 0x1c9; 0x1cc, 0x1cc; 0x1ce, 0x1ce; 0x1d0, 0x1d0; 0x1d2, 0x1d2; 0x1d4, 0x1d4; 0x1d6, 0x1d6; 0x1d8, 0x1d8; 0x1da, 0x1da; 0x1dc, 0x1dd; 0x1df, 0x1df; 0x1e1, 0x1e1; 0x1e3, 0x1e3; 0x1e5, 0x1e5; 0x1e7, 0x1e7; 0x1e9, 0x1e9; 0x1eb, 0x1eb; 0x1ed, 0x1ed; 0x1ef, 0x1f0; 0x1f3, 0x1f3; 0x1f5, 0x1f5; 0x1f9, 0x1f9; 0x1fb, 0x1fb; 0x1fd, 0x1fd; 0x1ff, 0x1ff; 0x201, 0x201; 0x203, 0x203; 0x205, 0x205; 0x207, 0x207; 0x209, 0x209; 0x20b, 0x20b; 0x20d, 0x20d; 0x20f, 0x20f; 0x211, 0x211; 0x213, 0x213; 0x215, 0x215; 0x217, 0x217; 0x219, 0x219; 0x21b, 0x21b; 0x21d, 0x21d; 0x21f, 0x21f; 0x221, 0x221; 0x223, 0x223; 0x225, 0x225; 0x227, 0x227; 0x229, 0x229; 0x22b, 0x22b; 0x22d, 0x22d; 0x22f, 0x22f; 0x231, 0x231; 0x233, 0x239; 0x23c, 0x23c; 0x23f, 0x240; 0x242, 0x242; 0x247, 0x247; 0x249, 0x249; 0x24b, 0x24b; 0x24d, 0x24d; 0x24f, 0x293; 0x296, 0x2b8; 0x2c0, 0x2c1; 0x2e0, 0x2e4; 0x345, 0x345; 0x371, 0x371; 0x373, 0x373; 0x377, 0x377; 0x37a, 0x37d; 0x390, 0x390; 0x3ac, 0x3ce; 0x3d0, 0x3d1; 0x3d5, 0x3d7; 0x3d9, 0x3d9; 0x3db, 0x3db; 0x3dd, 0x3dd; 0x3df, 0x3df; 0x3e1, 0x3e1; 0x3e3, 0x3e3; 0x3e5, 0x3e5; 0x3e7, 0x3e7; 0x3e9, 0x3e9; 0x3eb, 0x3eb; 0x3ed, 0x3ed; 0x3ef, 0x3f3; 0x3f5, 0x3f5; 0x3f8, 0x3f8; 0x3fb, 0x3fc; 0x430, 0x45f; 0x461, 0x461; 0x463, 0x463; 0x465, 0x465; 0x467, 0x467; 0x469, 0x469; 0x46b, 0x46b; 0x46d, 0x46d; 0x46f, 0x46f; 0x471, 0x471; 0x473, 0x473; 0x475, 0x475; 0x477, 0x477; 0x479, 0x479; 0x47b, 0x47b; 0x47d, 0x47d; 0x47f, 0x47f; 0x481, 0x481; 0x48b, 0x48b; 0x48d, 0x48d; 0x48f, 0x48f; 0x491, 0x491; 0x493, 0x493; 0x495, 0x495; 0x497, 0x497; 0x499, 0x499; 0x49b, 0x49b; 0x49d, 0x49d; 0x49f, 0x49f; 0x4a1, 0x4a1; 0x4a3, 0x4a3; 0x4a5, 0x4a5; 0x4a7, 0x4a7; 0x4a9, 0x4a9; 0x4ab, 0x4ab; 0x4ad, 0x4ad; 0x4af, 0x4af; 0x4b1, 0x4b1; 0x4b3, 0x4b3; 0x4b5, 0x4b5; 0x4b7, 0x4b7; 0x4b9, 0x4b9; 0x4bb, 0x4bb; 0x4bd, 0x4bd; 0x4bf, 0x4bf; 0x4c2, 0x4c2; 0x4c4, 0x4c4; 0x4c6, 0x4c6; 0x4c8, 0x4c8; 0x4ca, 0x4ca; 0x4cc, 0x4cc; 0x4ce, 0x4cf; 0x4d1, 0x4d1; 0x4d3, 0x4d3; 0x4d5, 0x4d5; 0x4d7, 0x4d7; 0x4d9, 0x4d9; 0x4db, 0x4db; 0x4dd, 0x4dd; 0x4df, 0x4df; 0x4e1, 0x4e1; 0x4e3, 0x4e3; 0x4e5, 0x4e5; 0x4e7, 0x4e7; 0x4e9, 0x4e9; 0x4eb, 0x4eb; 0x4ed, 0x4ed; 0x4ef, 0x4ef; 0x4f1, 0x4f1; 0x4f3, 0x4f3; 0x4f5, 0x4f5; 0x4f7, 0x4f7; 0x4f9, 0x4f9; 0x4fb, 0x4fb; 0x4fd, 0x4fd; 0x4ff, 0x4ff; 0x501, 0x501; 0x503, 0x503; 0x505, 0x505; 0x507, 0x507; 0x509, 0x509; 0x50b, 0x50b; 0x50d, 0x50d; 0x50f, 0x50f; 0x511, 0x511; 0x513, 0x513; 0x515, 0x515; 0x517, 0x517; 0x519, 0x519; 0x51b, 0x51b; 0x51d, 0x51d; 0x51f, 0x51f; 0x521, 0x521; 0x523, 0x523; 0x525, 0x525; 0x527, 0x527; 0x529, 0x529; 0x52b, 0x52b; 0x52d, 0x52d; 0x52f, 0x52f; 0x558, 0x558; 0x560, 0x588; 0x58b, 0x58c; 0x10d0, 0x10fa; 0x10fc, 0x10ff; 0x13f8, 0x13fd; 0x1c80, 0x1c88; 0x1c8a, 0x1c8a; 0x1d00, 0x1dbf; 0x1e01, 0x1e01; 0x1e03, 0x1e03; 0x1e05, 0x1e05; 0x1e07, 0x1e07; 0x1e09, 0x1e09; 0x1e0b, 0x1e0b; 0x1e0d, 0x1e0d; 0x1e0f, 0x1e0f; 0x1e11, 0x1e11; 0x1e13, 0x1e13; 0x1e15, 0x1e15; 0x1e17, 0x1e17; 0x1e19, 0x1e19; 0x1e1b, 0x1e1b; 0x1e1d, 0x1e1d; 0x1e1f, 0x1e1f; 0x1e21, 0x1e21; 0x1e23, 0x1e23; 0x1e25, 0x1e25; 0x1e27, 0x1e27; 0x1e29, 0x1e29; 0x1e2b, 0x1e2b; 0x1e2d, 0x1e2d; 0x1e2f, 0x1e2f; 0x1e31, 0x1e31; 0x1e33, 0x1e33; 0x1e35, 0x1e35; 0x1e37, 0x1e37; 0x1e39, 0x1e39; 0x1e3b, 0x1e3b; 0x1e3d, 0x1e3d; 0x1e3f, 0x1e3f; 0x1e41, 0x1e41; 0x1e43, 0x1e43; 0x1e45, 0x1e45; 0x1e47, 0x1e47; 0x1e49, 0x1e49; 0x1e4b, 0x1e4b; 0x1e4d, 0x1e4d; 0x1e4f, 0x1e4f; 0x1e51, 0x1e51; 0x1e53, 0x1e53; 0x1e55, 0x1e55; 0x1e57, 0x1e57; 0x1e59, 0x1e59; 0x1e5b, 0x1e5b; 0x1e5d, 0x1e5d; 0x1e5f, 0x1e5f; 0x1e61, 0x1e61; 0x1e63, 0x1e63; 0x1e65, 0x1e65; 0x1e67, 0x1e67; 0x1e69, 0x1e69; 0x1e6b, 0x1e6b; 0x1e6d, 0x1e6d; 0x1e6f, 0x1e6f; 0x1e71, 0x1e71; 0x1e73, 0x1e73; 0x1e75, 0x1e75; 0x1e77, 0x1e77; 0x1e79, 0x1e79; 0x1e7b, 0x1e7b; 0x1e7d, 0x1e7d; 0x1e7f, 0x1e7f; 0x1e81, 0x1e81; 0x1e83, 0x1e83; 0x1e85, 0x1e85; 0x1e87, 0x1e87; 0x1e89, 0x1e89; 0x1e8b, 0x1e8b; 0x1e8d, 0x1e8d; 0x1e8f, 0x1e8f; 0x1e91, 0x1e91; 0x1e93, 0x1e93; 0x1e95, 0x1e9d; 0x1e9f, 0x1e9f; 0x1ea1, 0x1ea1; 0x1ea3, 0x1ea3; 0x1ea5, 0x1ea5; 0x1ea7, 0x1ea7; 0x1ea9, 0x1ea9; 0x1eab, 0x1eab; 0x1ead, 0x1ead; 0x1eaf, 0x1eaf; 0x1eb1, 0x1eb1; 0x1eb3, 0x1eb3; 0x1eb5, 0x1eb5; 0x1eb7, 0x1eb7; 0x1eb9, 0x1eb9; 0x1ebb, 0x1ebb; 0x1ebd, 0x1ebd; 0x1ebf, 0x1ebf; 0x1ec1, 0x1ec1; 0x1ec3, 0x1ec3; 0x1ec5, 0x1ec5; 0x1ec7, 0x1ec7; 0x1ec9, 0x1ec9; 0x1ecb, 0x1ecb; 0x1ecd, 0x1ecd; 0x1ecf, 0x1ecf; 0x1ed1, 0x1ed1; 0x1ed3, 0x1ed3; 0x1ed5, 0x1ed5; 0x1ed7, 0x1ed7; 0x1ed9, 0x1ed9; 0x1edb, 0x1edb; 0x1edd, 0x1edd; 0x1edf, 0x1edf; 0x1ee1, 0x1ee1; 0x1ee3, 0x1ee3; 0x1ee5, 0x1ee5; 0x1ee7, 0x1ee7; 0x1ee9, 0x1ee9; 0x1eeb, 0x1eeb; 0x1eed, 0x1eed; 0x1eef, 0x1eef; 0x1ef1, 0x1ef1; 0x1ef3, 0x1ef3; 0x1ef5, 0x1ef5; 0x1ef7, 0x1ef7; 0x1ef9, 0x1ef9; 0x1efb, 0x1efb; 0x1efd, 0x1efd; 0x1eff, 0x1f07; 0x1f10, 0x1f15; 0x1f20, 0x1f27; 0x1f30, 0x1f37; 0x1f40, 0x1f45; 0x1f50, 0x1f57; 0x1f60, 0x1f67; 0x1f70, 0x1f7d; 0x1f80, 0x1f87; 0x1f90, 0x1f97; 0x1fa0, 0x1fa7; 0x1fb0, 0x1fb4; 0x1fb6, 0x1fb7; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fc7; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fd7; 0x1fe0, 0x1fe7; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ff7; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209f; 0x210a, 0x210a; 0x210e, 0x210f; 0x2113, 0x2113; 0x212f, 0x212f; 0x2134, 0x2134; 0x2139, 0x2139; 0x213c, 0x213d; 0x2146, 0x2149; 0x214e, 0x214e; 0x2170, 0x217f; 0x2184, 0x2184; 0x24d0, 0x24e9; 0x2c30, 0x2c5f; 0x2c61, 0x2c61; 0x2c65, 0x2c66; 0x2c68, 0x2c68; 0x2c6a, 0x2c6a; 0x2c6c, 0x2c6c; 0x2c71, 0x2c71; 0x2c73, 0x2c74; 0x2c76, 0x2c7d; 0x2c81, 0x2c81; 0x2c83, 0x2c83; 0x2c85, 0x2c85; 0x2c87, 0x2c87; 0x2c89, 0x2c89; 0x2c8b, 0x2c8b; 0x2c8d, 0x2c8d; 0x2c8f, 0x2c8f; 0x2c91, 0x2c91; 0x2c93, 0x2c93; 0x2c95, 0x2c95; 0x2c97, 0x2c97; 0x2c99, 0x2c99; 0x2c9b, 0x2c9b; 0x2c9d, 0x2c9d; 0x2c9f, 0x2c9f; 0x2ca1, 0x2ca1; 0x2ca3, 0x2ca3; 0x2ca5, 0x2ca5; 0x2ca7, 0x2ca7; 0x2ca9, 0x2ca9; 0x2cab, 0x2cab; 0x2cad, 0x2cad; 0x2caf, 0x2caf; 0x2cb1, 0x2cb1; 0x2cb3, 0x2cb3; 0x2cb5, 0x2cb5; 0x2cb7, 0x2cb7; 0x2cb9, 0x2cb9; 0x2cbb, 0x2cbb; 0x2cbd, 0x2cbd; 0x2cbf, 0x2cbf; 0x2cc1, 0x2cc1; 0x2cc3, 0x2cc3; 0x2cc5, 0x2cc5; 0x2cc7, 0x2cc7; 0x2cc9, 0x2cc9; 0x2ccb, 0x2ccb; 0x2ccd, 0x2ccd; 0x2ccf, 0x2ccf; 0x2cd1, 0x2cd1; 0x2cd3, 0x2cd3; 0x2cd5, 0x2cd5; 0x2cd7, 0x2cd7; 0x2cd9, 0x2cd9; 0x2cdb, 0x2cdb; 0x2cdd, 0x2cdd; 0x2cdf, 0x2cdf; 0x2ce1, 0x2ce1; 0x2ce3, 0x2ce4; 0x2cec, 0x2cec; 0x2cee, 0x2cee; 0x2cf3, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0xa641, 0xa641; 0xa643, 0xa643; 0xa645, 0xa645; 0xa647, 0xa647; 0xa649, 0xa649; 0xa64b, 0xa64b; 0xa64d, 0xa64d; 0xa64f, 0xa64f; 0xa651, 0xa651; 0xa653, 0xa653; 0xa655, 0xa655; 0xa657, 0xa657; 0xa659, 0xa659; 0xa65b, 0xa65b; 0xa65d, 0xa65d; 0xa65f, 0xa65f; 0xa661, 0xa661; 0xa663, 0xa663; 0xa665, 0xa665; 0xa667, 0xa667; 0xa669, 0xa669; 0xa66b, 0xa66b; 0xa66d, 0xa66d; 0xa681, 0xa681; 0xa683, 0xa683; 0xa685, 0xa685; 0xa687, 0xa687; 0xa689, 0xa689; 0xa68b, 0xa68b; 0xa68d, 0xa68d; 0xa68f, 0xa68f; 0xa691, 0xa691; 0xa693, 0xa693; 0xa695, 0xa695; 0xa697, 0xa697; 0xa699, 0xa699; 0xa69b, 0xa69d; 0xa723, 0xa723; 0xa725, 0xa725; 0xa727, 0xa727; 0xa729, 0xa729; 0xa72b, 0xa72b; 0xa72d, 0xa72d; 0xa72f, 0xa731; 0xa733, 0xa733; 0xa735, 0xa735; 0xa737, 0xa737; 0xa739, 0xa739; 0xa73b, 0xa73b; 0xa73d, 0xa73d; 0xa73f, 0xa73f; 0xa741, 0xa741; 0xa743, 0xa743; 0xa745, 0xa745; 0xa747, 0xa747; 0xa749, 0xa749; 0xa74b, 0xa74b; 0xa74d, 0xa74d; 0xa74f, 0xa74f; 0xa751, 0xa751; 0xa753, 0xa753; 0xa755, 0xa755; 0xa757, 0xa757; 0xa759, 0xa759; 0xa75b, 0xa75b; 0xa75d, 0xa75d; 0xa75f, 0xa75f; 0xa761, 0xa761; 0xa763, 0xa763; 0xa765, 0xa765; 0xa767, 0xa767; 0xa769, 0xa769; 0xa76b, 0xa76b; 0xa76d, 0xa76d; 0xa76f, 0xa778; 0xa77a, 0xa77a; 0xa77c, 0xa77c; 0xa77f, 0xa77f; 0xa781, 0xa781; 0xa783, 0xa783; 0xa785, 0xa785; 0xa787, 0xa787; 0xa78c, 0xa78c; 0xa78e, 0xa78e; 0xa791, 0xa791; 0xa793, 0xa795; 0xa797, 0xa797; 0xa799, 0xa799; 0xa79b, 0xa79b; 0xa79d, 0xa79d; 0xa79f, 0xa79f; 0xa7a1, 0xa7a1; 0xa7a3, 0xa7a3; 0xa7a5, 0xa7a5; 0xa7a7, 0xa7a7; 0xa7a9, 0xa7a9; 0xa7af, 0xa7af; 0xa7b5, 0xa7b5; 0xa7b7, 0xa7b7; 0xa7b9, 0xa7b9; 0xa7bb, 0xa7bb; 0xa7bd, 0xa7bd; 0xa7bf, 0xa7bf; 0xa7c1, 0xa7c1; 0xa7c3, 0xa7c3; 0xa7c8, 0xa7c8; 0xa7ca, 0xa7ca; 0xa7cd, 0xa7cd; 0xa7cf, 0xa7cf; 0xa7d1, 0xa7d1; 0xa7d3, 0xa7d3; 0xa7d5, 0xa7d5; 0xa7d7, 0xa7d7; 0xa7d9, 0xa7d9; 0xa7db, 0xa7db; 0xa7f1, 0xa7f4; 0xa7f6, 0xa7f6; 0xa7f8, 0xa7fa; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab70, 0xabbf; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xff41, 0xff5a; 0x10428, 0x1044f; 0x104d8, 0x104fb; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x10780, 0x10780; 0x10783, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10cc0, 0x10cf2; 0x10d70, 0x10d85; 0x118c0, 0x118df; 0x16e60, 0x16e7f; 0x16ebb, 0x16ed3; 0x1d41a, 0x1d433; 0x1d44e, 0x1d454; 0x1d456, 0x1d467; 0x1d482, 0x1d49b; 0x1d4b6, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d4cf; 0x1d4ea, 0x1d503; 0x1d51e, 0x1d537; 0x1d552, 0x1d56b; 0x1d586, 0x1d59f; 0x1d5ba, 0x1d5d3; 0x1d5ee, 0x1d607; 0x1d622, 0x1d63b; 0x1d656, 0x1d66f; 0x1d68a, 0x1d6a6; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6e1; 0x1d6fc, 0x1d714; 0x1d716, 0x1d71b; 0x1d736, 0x1d74e; 0x1d750, 0x1d755; 0x1d770, 0x1d788; 0x1d78a, 0x1d78f; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7c9; 0x1d7cb, 0x1d7cb; 0x1df00, 0x1df09; 0x1df0b, 0x1df3f; 0x1df41, 0x1df47; 0x1df49, 0x1df49; 0x1df4b, 0x1df4c; 0x1df4e, 0x1df50; 0x1df52, 0x1df67; 0x1df69, 0x1df69; 0x1df6b, 0x1df6b; 0x1df6d, 0x1df6d; 0x1df6f, 0x1df71; 0x1df73, 0x1df73; 0x1df75, 0x1df75; 0x1df77, 0x1df77; 0x1df79, 0x1df79; 0x1df7b, 0x1df7b; 0x1df7d, 0x1df7d; 0x1df7f, 0x1df7f; 0x1df90, 0x1df96; 0x1dfcd, 0x1dfff; 0x1e030, 0x1e06d; 0x1e922, 0x1e943] let math = Cset.of_list [0x2b, 0x2b; 0x3c, 0x3e; 0x5e, 0x5e; 0x7c, 0x7c; 0x7e, 0x7e; 0xac, 0xac; 0xb1, 0xb1; 0xd7, 0xd7; 0xf7, 0xf7; 0x3d0, 0x3d2; 0x3d5, 0x3d5; 0x3f0, 0x3f1; 0x3f4, 0x3f6; 0x606, 0x608; 0x2016, 0x2016; 0x2032, 0x2034; 0x2040, 0x2040; 0x2044, 0x2044; 0x2052, 0x2052; 0x2061, 0x2064; 0x207a, 0x207e; 0x208a, 0x208e; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20e6; 0x20eb, 0x20ef; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2128, 0x2129; 0x212c, 0x212d; 0x212f, 0x2131; 0x2133, 0x2138; 0x213c, 0x2149; 0x214b, 0x214b; 0x2190, 0x21a7; 0x21a9, 0x21ae; 0x21b0, 0x21b1; 0x21b6, 0x21b7; 0x21bc, 0x21db; 0x21dd, 0x21dd; 0x21e4, 0x21e5; 0x21f4, 0x22ff; 0x2308, 0x230b; 0x2320, 0x2321; 0x237c, 0x237c; 0x239b, 0x23b5; 0x23b7, 0x23b7; 0x23d0, 0x23d0; 0x23dc, 0x23e2; 0x25a0, 0x25a1; 0x25ae, 0x25b7; 0x25bc, 0x25c1; 0x25c6, 0x25c7; 0x25ca, 0x25cb; 0x25cf, 0x25d3; 0x25e2, 0x25e2; 0x25e4, 0x25e4; 0x25e7, 0x25ec; 0x25f8, 0x25ff; 0x2605, 0x2606; 0x2640, 0x2640; 0x2642, 0x2642; 0x2660, 0x2663; 0x266d, 0x266f; 0x27c0, 0x27ff; 0x2900, 0x2aff; 0x2b30, 0x2b44; 0x2b47, 0x2b4c; 0xfb29, 0xfb29; 0xfe61, 0xfe66; 0xfe68, 0xfe68; 0xff0b, 0xff0b; 0xff1c, 0xff1e; 0xff3c, 0xff3c; 0xff3e, 0xff3e; 0xff5c, 0xff5c; 0xff5e, 0xff5e; 0xffe2, 0xffe2; 0xffe9, 0xffec; 0x10d8e, 0x10d8f; 0x1cedd, 0x1cedf; 0x1cef0, 0x1cefd; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1db00, 0x1db1c; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1eef0, 0x1eef1; 0x1f7db, 0x1f7db; 0x1f7f1, 0x1f7ff; 0x1f8d0, 0x1f8d8] let other_alphabetic = Cset.of_list [0x345, 0x345; 0x363, 0x36f; 0x5b0, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c9; 0x610, 0x61a; 0x64b, 0x657; 0x659, 0x65f; 0x670, 0x670; 0x6d6, 0x6dc; 0x6e1, 0x6e4; 0x6e7, 0x6e8; 0x6ed, 0x6ed; 0x711, 0x711; 0x730, 0x73f; 0x7a6, 0x7b0; 0x816, 0x817; 0x81b, 0x823; 0x825, 0x827; 0x829, 0x82c; 0x897, 0x897; 0x8d4, 0x8df; 0x8e3, 0x8e9; 0x8f0, 0x903; 0x93a, 0x93b; 0x93e, 0x94c; 0x94e, 0x94f; 0x955, 0x957; 0x962, 0x963; 0x981, 0x983; 0x9be, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9cc; 0x9d7, 0x9d7; 0x9e2, 0x9e3; 0xa01, 0xa03; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4c; 0xa51, 0xa51; 0xa70, 0xa71; 0xa75, 0xa75; 0xa81, 0xa83; 0xabe, 0xac5; 0xac7, 0xac9; 0xacb, 0xacc; 0xae2, 0xae3; 0xafa, 0xafc; 0xb01, 0xb03; 0xb3e, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4c; 0xb56, 0xb57; 0xb62, 0xb63; 0xb82, 0xb82; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcc; 0xbd7, 0xbd7; 0xc00, 0xc04; 0xc3e, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4c; 0xc55, 0xc56; 0xc62, 0xc63; 0xc81, 0xc83; 0xcbe, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccc; 0xcd5, 0xcd6; 0xce2, 0xce3; 0xcf3, 0xcf3; 0xd00, 0xd03; 0xd3e, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4c; 0xd57, 0xd57; 0xd62, 0xd63; 0xd81, 0xd83; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xdf2, 0xdf3; 0xe31, 0xe31; 0xe34, 0xe3a; 0xe4d, 0xe4d; 0xeb1, 0xeb1; 0xeb4, 0xeb9; 0xebb, 0xebc; 0xecd, 0xecd; 0xf71, 0xf83; 0xf8d, 0xf97; 0xf99, 0xfbc; 0x102b, 0x1036; 0x1038, 0x1038; 0x103b, 0x103e; 0x1056, 0x1059; 0x105e, 0x1060; 0x1062, 0x1064; 0x1067, 0x106d; 0x1071, 0x1074; 0x1082, 0x108d; 0x108f, 0x108f; 0x109a, 0x109d; 0x1712, 0x1713; 0x1732, 0x1733; 0x1752, 0x1753; 0x1772, 0x1773; 0x17b6, 0x17c8; 0x1885, 0x1886; 0x18a9, 0x18a9; 0x1920, 0x192b; 0x1930, 0x1938; 0x1a17, 0x1a1b; 0x1a55, 0x1a5e; 0x1a61, 0x1a74; 0x1abf, 0x1ac0; 0x1acc, 0x1ace; 0x1b00, 0x1b04; 0x1b35, 0x1b43; 0x1b80, 0x1b82; 0x1ba1, 0x1ba9; 0x1bac, 0x1bad; 0x1be7, 0x1bf1; 0x1c24, 0x1c36; 0x1dd3, 0x1df4; 0x24b6, 0x24e9; 0x2de0, 0x2dff; 0xa674, 0xa67b; 0xa69e, 0xa69f; 0xa802, 0xa802; 0xa80b, 0xa80b; 0xa823, 0xa827; 0xa880, 0xa881; 0xa8b4, 0xa8c3; 0xa8c5, 0xa8c5; 0xa8ff, 0xa8ff; 0xa926, 0xa92a; 0xa947, 0xa952; 0xa980, 0xa983; 0xa9b4, 0xa9bf; 0xa9e5, 0xa9e5; 0xaa29, 0xaa36; 0xaa43, 0xaa43; 0xaa4c, 0xaa4d; 0xaa7b, 0xaa7d; 0xaab0, 0xaab0; 0xaab2, 0xaab4; 0xaab7, 0xaab8; 0xaabe, 0xaabe; 0xaaeb, 0xaaef; 0xaaf5, 0xaaf5; 0xabe3, 0xabea; 0xfb1e, 0xfb1e; 0x10376, 0x1037a; 0x10a01, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a0f; 0x10d24, 0x10d27; 0x10d69, 0x10d69; 0x10eab, 0x10eac; 0x10ecb, 0x10ecd; 0x10ef3, 0x10ef3; 0x10ef5, 0x10ef5; 0x10ef7, 0x10ef8; 0x10efa, 0x10efc; 0x11000, 0x11002; 0x11038, 0x11045; 0x11073, 0x11074; 0x11080, 0x11082; 0x110b0, 0x110b8; 0x110c2, 0x110c2; 0x11100, 0x11102; 0x11127, 0x11132; 0x11145, 0x11146; 0x11180, 0x11182; 0x111b3, 0x111bf; 0x111ce, 0x111cf; 0x1122c, 0x11234; 0x11237, 0x11237; 0x1123e, 0x1123e; 0x11241, 0x11241; 0x112df, 0x112e8; 0x11300, 0x11303; 0x1133e, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134c; 0x11357, 0x11357; 0x11362, 0x11363; 0x113b8, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113cd; 0x11435, 0x11441; 0x11443, 0x11445; 0x114b0, 0x114c1; 0x115af, 0x115b5; 0x115b8, 0x115be; 0x115dc, 0x115dd; 0x11630, 0x1163e; 0x11640, 0x11640; 0x116ab, 0x116b5; 0x1171d, 0x1172a; 0x1182c, 0x11838; 0x11930, 0x11935; 0x11937, 0x11938; 0x1193b, 0x1193c; 0x11940, 0x11940; 0x11942, 0x11942; 0x119d1, 0x119d7; 0x119da, 0x119df; 0x119e4, 0x119e4; 0x11a01, 0x11a0a; 0x11a35, 0x11a39; 0x11a3b, 0x11a3e; 0x11a51, 0x11a5b; 0x11a8a, 0x11a97; 0x11b60, 0x11b67; 0x11c2f, 0x11c36; 0x11c38, 0x11c3e; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d31, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d41; 0x11d43, 0x11d43; 0x11d47, 0x11d47; 0x11d8a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d96; 0x11df0, 0x11df0; 0x11ef3, 0x11ef6; 0x11f00, 0x11f01; 0x11f03, 0x11f03; 0x11f34, 0x11f3a; 0x11f3e, 0x11f40; 0x1611e, 0x1612e; 0x16f4f, 0x16f4f; 0x16f51, 0x16f87; 0x16f8f, 0x16f92; 0x16ff0, 0x16ff1; 0x1bc9e, 0x1bc9e; 0x1e000, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e08f, 0x1e08f; 0x1e6e3, 0x1e6e3; 0x1e6e6, 0x1e6e6; 0x1e6ee, 0x1e6ef; 0x1e6f5, 0x1e6f5; 0x1e947, 0x1e947; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189] let other_lowercase = Cset.of_list [0xaa, 0xaa; 0xba, 0xba; 0x2b0, 0x2b8; 0x2c0, 0x2c1; 0x2e0, 0x2e4; 0x345, 0x345; 0x37a, 0x37a; 0x558, 0x558; 0x58b, 0x58c; 0x10fc, 0x10fc; 0x1d2c, 0x1d6a; 0x1d78, 0x1d78; 0x1d9b, 0x1dbf; 0x2071, 0x2071; 0x207f, 0x207f; 0x2090, 0x209f; 0x2170, 0x217f; 0x24d0, 0x24e9; 0x2c7c, 0x2c7d; 0xa69c, 0xa69d; 0xa770, 0xa770; 0xa7f1, 0xa7f4; 0xa7f8, 0xa7f9; 0xab5c, 0xab5f; 0xab69, 0xab69; 0x10780, 0x10780; 0x10783, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x1dfcd, 0x1dfff; 0x1e030, 0x1e06d] let other_math = Cset.of_list [0x5e, 0x5e; 0x3d0, 0x3d2; 0x3d5, 0x3d5; 0x3f0, 0x3f1; 0x3f4, 0x3f5; 0x2016, 0x2016; 0x2032, 0x2034; 0x2040, 0x2040; 0x2061, 0x2064; 0x207d, 0x207e; 0x208d, 0x208e; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20e6; 0x20eb, 0x20ef; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2128, 0x2129; 0x212c, 0x212d; 0x212f, 0x2131; 0x2133, 0x2138; 0x213c, 0x213f; 0x2145, 0x2149; 0x2195, 0x2199; 0x219c, 0x219f; 0x21a1, 0x21a2; 0x21a4, 0x21a5; 0x21a7, 0x21a7; 0x21a9, 0x21ad; 0x21b0, 0x21b1; 0x21b6, 0x21b7; 0x21bc, 0x21cd; 0x21d0, 0x21d1; 0x21d3, 0x21d3; 0x21d5, 0x21db; 0x21dd, 0x21dd; 0x21e4, 0x21e5; 0x2308, 0x230b; 0x23b4, 0x23b5; 0x23b7, 0x23b7; 0x23d0, 0x23d0; 0x23e2, 0x23e2; 0x25a0, 0x25a1; 0x25ae, 0x25b6; 0x25bc, 0x25c0; 0x25c6, 0x25c7; 0x25ca, 0x25cb; 0x25cf, 0x25d3; 0x25e2, 0x25e2; 0x25e4, 0x25e4; 0x25e7, 0x25ec; 0x2605, 0x2606; 0x2640, 0x2640; 0x2642, 0x2642; 0x2660, 0x2663; 0x266d, 0x266e; 0x27c5, 0x27c6; 0x27e6, 0x27ef; 0x2983, 0x2998; 0x29d8, 0x29db; 0x29fc, 0x29fd; 0xfe61, 0xfe61; 0xfe63, 0xfe63; 0xfe68, 0xfe68; 0xff3c, 0xff3c; 0xff3e, 0xff3e; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1f7db, 0x1f7db; 0x1f7f1, 0x1f7ff] let other_uppercase = Cset.of_list [0x2160, 0x216f; 0x24b6, 0x24cf; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189] let uppercase = Cset.of_list [0x41, 0x5a; 0xc0, 0xd6; 0xd8, 0xde; 0x100, 0x100; 0x102, 0x102; 0x104, 0x104; 0x106, 0x106; 0x108, 0x108; 0x10a, 0x10a; 0x10c, 0x10c; 0x10e, 0x10e; 0x110, 0x110; 0x112, 0x112; 0x114, 0x114; 0x116, 0x116; 0x118, 0x118; 0x11a, 0x11a; 0x11c, 0x11c; 0x11e, 0x11e; 0x120, 0x120; 0x122, 0x122; 0x124, 0x124; 0x126, 0x126; 0x128, 0x128; 0x12a, 0x12a; 0x12c, 0x12c; 0x12e, 0x12e; 0x130, 0x130; 0x132, 0x132; 0x134, 0x134; 0x136, 0x136; 0x139, 0x139; 0x13b, 0x13b; 0x13d, 0x13d; 0x13f, 0x13f; 0x141, 0x141; 0x143, 0x143; 0x145, 0x145; 0x147, 0x147; 0x14a, 0x14a; 0x14c, 0x14c; 0x14e, 0x14e; 0x150, 0x150; 0x152, 0x152; 0x154, 0x154; 0x156, 0x156; 0x158, 0x158; 0x15a, 0x15a; 0x15c, 0x15c; 0x15e, 0x15e; 0x160, 0x160; 0x162, 0x162; 0x164, 0x164; 0x166, 0x166; 0x168, 0x168; 0x16a, 0x16a; 0x16c, 0x16c; 0x16e, 0x16e; 0x170, 0x170; 0x172, 0x172; 0x174, 0x174; 0x176, 0x176; 0x178, 0x179; 0x17b, 0x17b; 0x17d, 0x17d; 0x181, 0x182; 0x184, 0x184; 0x186, 0x187; 0x189, 0x18b; 0x18e, 0x191; 0x193, 0x194; 0x196, 0x198; 0x19c, 0x19d; 0x19f, 0x1a0; 0x1a2, 0x1a2; 0x1a4, 0x1a4; 0x1a6, 0x1a7; 0x1a9, 0x1a9; 0x1ac, 0x1ac; 0x1ae, 0x1af; 0x1b1, 0x1b3; 0x1b5, 0x1b5; 0x1b7, 0x1b8; 0x1bc, 0x1bc; 0x1c4, 0x1c4; 0x1c7, 0x1c7; 0x1ca, 0x1ca; 0x1cd, 0x1cd; 0x1cf, 0x1cf; 0x1d1, 0x1d1; 0x1d3, 0x1d3; 0x1d5, 0x1d5; 0x1d7, 0x1d7; 0x1d9, 0x1d9; 0x1db, 0x1db; 0x1de, 0x1de; 0x1e0, 0x1e0; 0x1e2, 0x1e2; 0x1e4, 0x1e4; 0x1e6, 0x1e6; 0x1e8, 0x1e8; 0x1ea, 0x1ea; 0x1ec, 0x1ec; 0x1ee, 0x1ee; 0x1f1, 0x1f1; 0x1f4, 0x1f4; 0x1f6, 0x1f8; 0x1fa, 0x1fa; 0x1fc, 0x1fc; 0x1fe, 0x1fe; 0x200, 0x200; 0x202, 0x202; 0x204, 0x204; 0x206, 0x206; 0x208, 0x208; 0x20a, 0x20a; 0x20c, 0x20c; 0x20e, 0x20e; 0x210, 0x210; 0x212, 0x212; 0x214, 0x214; 0x216, 0x216; 0x218, 0x218; 0x21a, 0x21a; 0x21c, 0x21c; 0x21e, 0x21e; 0x220, 0x220; 0x222, 0x222; 0x224, 0x224; 0x226, 0x226; 0x228, 0x228; 0x22a, 0x22a; 0x22c, 0x22c; 0x22e, 0x22e; 0x230, 0x230; 0x232, 0x232; 0x23a, 0x23b; 0x23d, 0x23e; 0x241, 0x241; 0x243, 0x246; 0x248, 0x248; 0x24a, 0x24a; 0x24c, 0x24c; 0x24e, 0x24e; 0x370, 0x370; 0x372, 0x372; 0x376, 0x376; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x38f; 0x391, 0x3a1; 0x3a3, 0x3ab; 0x3cf, 0x3cf; 0x3d2, 0x3d4; 0x3d8, 0x3d8; 0x3da, 0x3da; 0x3dc, 0x3dc; 0x3de, 0x3de; 0x3e0, 0x3e0; 0x3e2, 0x3e2; 0x3e4, 0x3e4; 0x3e6, 0x3e6; 0x3e8, 0x3e8; 0x3ea, 0x3ea; 0x3ec, 0x3ec; 0x3ee, 0x3ee; 0x3f4, 0x3f4; 0x3f7, 0x3f7; 0x3f9, 0x3fa; 0x3fd, 0x42f; 0x460, 0x460; 0x462, 0x462; 0x464, 0x464; 0x466, 0x466; 0x468, 0x468; 0x46a, 0x46a; 0x46c, 0x46c; 0x46e, 0x46e; 0x470, 0x470; 0x472, 0x472; 0x474, 0x474; 0x476, 0x476; 0x478, 0x478; 0x47a, 0x47a; 0x47c, 0x47c; 0x47e, 0x47e; 0x480, 0x480; 0x48a, 0x48a; 0x48c, 0x48c; 0x48e, 0x48e; 0x490, 0x490; 0x492, 0x492; 0x494, 0x494; 0x496, 0x496; 0x498, 0x498; 0x49a, 0x49a; 0x49c, 0x49c; 0x49e, 0x49e; 0x4a0, 0x4a0; 0x4a2, 0x4a2; 0x4a4, 0x4a4; 0x4a6, 0x4a6; 0x4a8, 0x4a8; 0x4aa, 0x4aa; 0x4ac, 0x4ac; 0x4ae, 0x4ae; 0x4b0, 0x4b0; 0x4b2, 0x4b2; 0x4b4, 0x4b4; 0x4b6, 0x4b6; 0x4b8, 0x4b8; 0x4ba, 0x4ba; 0x4bc, 0x4bc; 0x4be, 0x4be; 0x4c0, 0x4c1; 0x4c3, 0x4c3; 0x4c5, 0x4c5; 0x4c7, 0x4c7; 0x4c9, 0x4c9; 0x4cb, 0x4cb; 0x4cd, 0x4cd; 0x4d0, 0x4d0; 0x4d2, 0x4d2; 0x4d4, 0x4d4; 0x4d6, 0x4d6; 0x4d8, 0x4d8; 0x4da, 0x4da; 0x4dc, 0x4dc; 0x4de, 0x4de; 0x4e0, 0x4e0; 0x4e2, 0x4e2; 0x4e4, 0x4e4; 0x4e6, 0x4e6; 0x4e8, 0x4e8; 0x4ea, 0x4ea; 0x4ec, 0x4ec; 0x4ee, 0x4ee; 0x4f0, 0x4f0; 0x4f2, 0x4f2; 0x4f4, 0x4f4; 0x4f6, 0x4f6; 0x4f8, 0x4f8; 0x4fa, 0x4fa; 0x4fc, 0x4fc; 0x4fe, 0x4fe; 0x500, 0x500; 0x502, 0x502; 0x504, 0x504; 0x506, 0x506; 0x508, 0x508; 0x50a, 0x50a; 0x50c, 0x50c; 0x50e, 0x50e; 0x510, 0x510; 0x512, 0x512; 0x514, 0x514; 0x516, 0x516; 0x518, 0x518; 0x51a, 0x51a; 0x51c, 0x51c; 0x51e, 0x51e; 0x520, 0x520; 0x522, 0x522; 0x524, 0x524; 0x526, 0x526; 0x528, 0x528; 0x52a, 0x52a; 0x52c, 0x52c; 0x52e, 0x52e; 0x531, 0x556; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x13a0, 0x13f5; 0x1c89, 0x1c89; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1e00, 0x1e00; 0x1e02, 0x1e02; 0x1e04, 0x1e04; 0x1e06, 0x1e06; 0x1e08, 0x1e08; 0x1e0a, 0x1e0a; 0x1e0c, 0x1e0c; 0x1e0e, 0x1e0e; 0x1e10, 0x1e10; 0x1e12, 0x1e12; 0x1e14, 0x1e14; 0x1e16, 0x1e16; 0x1e18, 0x1e18; 0x1e1a, 0x1e1a; 0x1e1c, 0x1e1c; 0x1e1e, 0x1e1e; 0x1e20, 0x1e20; 0x1e22, 0x1e22; 0x1e24, 0x1e24; 0x1e26, 0x1e26; 0x1e28, 0x1e28; 0x1e2a, 0x1e2a; 0x1e2c, 0x1e2c; 0x1e2e, 0x1e2e; 0x1e30, 0x1e30; 0x1e32, 0x1e32; 0x1e34, 0x1e34; 0x1e36, 0x1e36; 0x1e38, 0x1e38; 0x1e3a, 0x1e3a; 0x1e3c, 0x1e3c; 0x1e3e, 0x1e3e; 0x1e40, 0x1e40; 0x1e42, 0x1e42; 0x1e44, 0x1e44; 0x1e46, 0x1e46; 0x1e48, 0x1e48; 0x1e4a, 0x1e4a; 0x1e4c, 0x1e4c; 0x1e4e, 0x1e4e; 0x1e50, 0x1e50; 0x1e52, 0x1e52; 0x1e54, 0x1e54; 0x1e56, 0x1e56; 0x1e58, 0x1e58; 0x1e5a, 0x1e5a; 0x1e5c, 0x1e5c; 0x1e5e, 0x1e5e; 0x1e60, 0x1e60; 0x1e62, 0x1e62; 0x1e64, 0x1e64; 0x1e66, 0x1e66; 0x1e68, 0x1e68; 0x1e6a, 0x1e6a; 0x1e6c, 0x1e6c; 0x1e6e, 0x1e6e; 0x1e70, 0x1e70; 0x1e72, 0x1e72; 0x1e74, 0x1e74; 0x1e76, 0x1e76; 0x1e78, 0x1e78; 0x1e7a, 0x1e7a; 0x1e7c, 0x1e7c; 0x1e7e, 0x1e7e; 0x1e80, 0x1e80; 0x1e82, 0x1e82; 0x1e84, 0x1e84; 0x1e86, 0x1e86; 0x1e88, 0x1e88; 0x1e8a, 0x1e8a; 0x1e8c, 0x1e8c; 0x1e8e, 0x1e8e; 0x1e90, 0x1e90; 0x1e92, 0x1e92; 0x1e94, 0x1e94; 0x1e9e, 0x1e9e; 0x1ea0, 0x1ea0; 0x1ea2, 0x1ea2; 0x1ea4, 0x1ea4; 0x1ea6, 0x1ea6; 0x1ea8, 0x1ea8; 0x1eaa, 0x1eaa; 0x1eac, 0x1eac; 0x1eae, 0x1eae; 0x1eb0, 0x1eb0; 0x1eb2, 0x1eb2; 0x1eb4, 0x1eb4; 0x1eb6, 0x1eb6; 0x1eb8, 0x1eb8; 0x1eba, 0x1eba; 0x1ebc, 0x1ebc; 0x1ebe, 0x1ebe; 0x1ec0, 0x1ec0; 0x1ec2, 0x1ec2; 0x1ec4, 0x1ec4; 0x1ec6, 0x1ec6; 0x1ec8, 0x1ec8; 0x1eca, 0x1eca; 0x1ecc, 0x1ecc; 0x1ece, 0x1ece; 0x1ed0, 0x1ed0; 0x1ed2, 0x1ed2; 0x1ed4, 0x1ed4; 0x1ed6, 0x1ed6; 0x1ed8, 0x1ed8; 0x1eda, 0x1eda; 0x1edc, 0x1edc; 0x1ede, 0x1ede; 0x1ee0, 0x1ee0; 0x1ee2, 0x1ee2; 0x1ee4, 0x1ee4; 0x1ee6, 0x1ee6; 0x1ee8, 0x1ee8; 0x1eea, 0x1eea; 0x1eec, 0x1eec; 0x1eee, 0x1eee; 0x1ef0, 0x1ef0; 0x1ef2, 0x1ef2; 0x1ef4, 0x1ef4; 0x1ef6, 0x1ef6; 0x1ef8, 0x1ef8; 0x1efa, 0x1efa; 0x1efc, 0x1efc; 0x1efe, 0x1efe; 0x1f08, 0x1f0f; 0x1f18, 0x1f1d; 0x1f28, 0x1f2f; 0x1f38, 0x1f3f; 0x1f48, 0x1f4d; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f5f; 0x1f68, 0x1f6f; 0x1fb8, 0x1fbb; 0x1fc8, 0x1fcb; 0x1fd8, 0x1fdb; 0x1fe8, 0x1fec; 0x1ff8, 0x1ffb; 0x2102, 0x2102; 0x2107, 0x2107; 0x210b, 0x210d; 0x2110, 0x2112; 0x2115, 0x2115; 0x2119, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x212d; 0x2130, 0x2133; 0x213e, 0x213f; 0x2145, 0x2145; 0x2160, 0x216f; 0x2183, 0x2183; 0x24b6, 0x24cf; 0x2c00, 0x2c2f; 0x2c60, 0x2c60; 0x2c62, 0x2c64; 0x2c67, 0x2c67; 0x2c69, 0x2c69; 0x2c6b, 0x2c6b; 0x2c6d, 0x2c70; 0x2c72, 0x2c72; 0x2c75, 0x2c75; 0x2c7e, 0x2c80; 0x2c82, 0x2c82; 0x2c84, 0x2c84; 0x2c86, 0x2c86; 0x2c88, 0x2c88; 0x2c8a, 0x2c8a; 0x2c8c, 0x2c8c; 0x2c8e, 0x2c8e; 0x2c90, 0x2c90; 0x2c92, 0x2c92; 0x2c94, 0x2c94; 0x2c96, 0x2c96; 0x2c98, 0x2c98; 0x2c9a, 0x2c9a; 0x2c9c, 0x2c9c; 0x2c9e, 0x2c9e; 0x2ca0, 0x2ca0; 0x2ca2, 0x2ca2; 0x2ca4, 0x2ca4; 0x2ca6, 0x2ca6; 0x2ca8, 0x2ca8; 0x2caa, 0x2caa; 0x2cac, 0x2cac; 0x2cae, 0x2cae; 0x2cb0, 0x2cb0; 0x2cb2, 0x2cb2; 0x2cb4, 0x2cb4; 0x2cb6, 0x2cb6; 0x2cb8, 0x2cb8; 0x2cba, 0x2cba; 0x2cbc, 0x2cbc; 0x2cbe, 0x2cbe; 0x2cc0, 0x2cc0; 0x2cc2, 0x2cc2; 0x2cc4, 0x2cc4; 0x2cc6, 0x2cc6; 0x2cc8, 0x2cc8; 0x2cca, 0x2cca; 0x2ccc, 0x2ccc; 0x2cce, 0x2cce; 0x2cd0, 0x2cd0; 0x2cd2, 0x2cd2; 0x2cd4, 0x2cd4; 0x2cd6, 0x2cd6; 0x2cd8, 0x2cd8; 0x2cda, 0x2cda; 0x2cdc, 0x2cdc; 0x2cde, 0x2cde; 0x2ce0, 0x2ce0; 0x2ce2, 0x2ce2; 0x2ceb, 0x2ceb; 0x2ced, 0x2ced; 0x2cf2, 0x2cf2; 0xa640, 0xa640; 0xa642, 0xa642; 0xa644, 0xa644; 0xa646, 0xa646; 0xa648, 0xa648; 0xa64a, 0xa64a; 0xa64c, 0xa64c; 0xa64e, 0xa64e; 0xa650, 0xa650; 0xa652, 0xa652; 0xa654, 0xa654; 0xa656, 0xa656; 0xa658, 0xa658; 0xa65a, 0xa65a; 0xa65c, 0xa65c; 0xa65e, 0xa65e; 0xa660, 0xa660; 0xa662, 0xa662; 0xa664, 0xa664; 0xa666, 0xa666; 0xa668, 0xa668; 0xa66a, 0xa66a; 0xa66c, 0xa66c; 0xa680, 0xa680; 0xa682, 0xa682; 0xa684, 0xa684; 0xa686, 0xa686; 0xa688, 0xa688; 0xa68a, 0xa68a; 0xa68c, 0xa68c; 0xa68e, 0xa68e; 0xa690, 0xa690; 0xa692, 0xa692; 0xa694, 0xa694; 0xa696, 0xa696; 0xa698, 0xa698; 0xa69a, 0xa69a; 0xa722, 0xa722; 0xa724, 0xa724; 0xa726, 0xa726; 0xa728, 0xa728; 0xa72a, 0xa72a; 0xa72c, 0xa72c; 0xa72e, 0xa72e; 0xa732, 0xa732; 0xa734, 0xa734; 0xa736, 0xa736; 0xa738, 0xa738; 0xa73a, 0xa73a; 0xa73c, 0xa73c; 0xa73e, 0xa73e; 0xa740, 0xa740; 0xa742, 0xa742; 0xa744, 0xa744; 0xa746, 0xa746; 0xa748, 0xa748; 0xa74a, 0xa74a; 0xa74c, 0xa74c; 0xa74e, 0xa74e; 0xa750, 0xa750; 0xa752, 0xa752; 0xa754, 0xa754; 0xa756, 0xa756; 0xa758, 0xa758; 0xa75a, 0xa75a; 0xa75c, 0xa75c; 0xa75e, 0xa75e; 0xa760, 0xa760; 0xa762, 0xa762; 0xa764, 0xa764; 0xa766, 0xa766; 0xa768, 0xa768; 0xa76a, 0xa76a; 0xa76c, 0xa76c; 0xa76e, 0xa76e; 0xa779, 0xa779; 0xa77b, 0xa77b; 0xa77d, 0xa77e; 0xa780, 0xa780; 0xa782, 0xa782; 0xa784, 0xa784; 0xa786, 0xa786; 0xa78b, 0xa78b; 0xa78d, 0xa78d; 0xa790, 0xa790; 0xa792, 0xa792; 0xa796, 0xa796; 0xa798, 0xa798; 0xa79a, 0xa79a; 0xa79c, 0xa79c; 0xa79e, 0xa79e; 0xa7a0, 0xa7a0; 0xa7a2, 0xa7a2; 0xa7a4, 0xa7a4; 0xa7a6, 0xa7a6; 0xa7a8, 0xa7a8; 0xa7aa, 0xa7ae; 0xa7b0, 0xa7b4; 0xa7b6, 0xa7b6; 0xa7b8, 0xa7b8; 0xa7ba, 0xa7ba; 0xa7bc, 0xa7bc; 0xa7be, 0xa7be; 0xa7c0, 0xa7c0; 0xa7c2, 0xa7c2; 0xa7c4, 0xa7c7; 0xa7c9, 0xa7c9; 0xa7cb, 0xa7cc; 0xa7ce, 0xa7ce; 0xa7d0, 0xa7d0; 0xa7d2, 0xa7d2; 0xa7d4, 0xa7d4; 0xa7d6, 0xa7d6; 0xa7d8, 0xa7d8; 0xa7da, 0xa7da; 0xa7dc, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f5, 0xa7f5; 0xab6c, 0xab6d; 0xff21, 0xff3a; 0x10400, 0x10427; 0x104b0, 0x104d3; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10c80, 0x10cb2; 0x10d50, 0x10d65; 0x118a0, 0x118bf; 0x16e40, 0x16e5f; 0x16ea0, 0x16eb8; 0x1d400, 0x1d419; 0x1d434, 0x1d44d; 0x1d468, 0x1d481; 0x1d49c, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b5; 0x1d4d0, 0x1d4e9; 0x1d504, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d538, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d56c, 0x1d585; 0x1d5a0, 0x1d5b9; 0x1d5d4, 0x1d5ed; 0x1d608, 0x1d621; 0x1d63c, 0x1d655; 0x1d670, 0x1d689; 0x1d6a8, 0x1d6c0; 0x1d6e2, 0x1d6fa; 0x1d71c, 0x1d734; 0x1d756, 0x1d76e; 0x1d790, 0x1d7a8; 0x1d7ca, 0x1d7ca; 0x1df40, 0x1df40; 0x1df48, 0x1df48; 0x1df4a, 0x1df4a; 0x1df4d, 0x1df4d; 0x1df51, 0x1df51; 0x1df68, 0x1df68; 0x1df6a, 0x1df6a; 0x1df6c, 0x1df6c; 0x1df6e, 0x1df6e; 0x1df72, 0x1df72; 0x1df74, 0x1df74; 0x1df76, 0x1df76; 0x1df78, 0x1df78; 0x1df7a, 0x1df7a; 0x1df7c, 0x1df7c; 0x1df7e, 0x1df7e; 0x1e900, 0x1e921; 0x1f130, 0x1f149; 0x1f150, 0x1f169; 0x1f170, 0x1f189] let white_space = Cset.of_list [0x9, 0xd; 0x20, 0x20; 0x85, 0x85; 0xa0, 0xa0; 0x1680, 0x1680; 0x2000, 0x200a; 0x2028, 0x2029; 0x202f, 0x202f; 0x205f, 0x205f; 0x3000, 0x3000] let xid_continue = Cset.of_list [0x30, 0x39; 0x41, 0x5a; 0x5f, 0x5f; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xb7, 0xb7; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x300, 0x374; 0x376, 0x377; 0x37b, 0x37d; 0x37f, 0x37f; 0x386, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x483, 0x487; 0x48a, 0x52f; 0x531, 0x556; 0x558, 0x559; 0x560, 0x588; 0x58b, 0x58c; 0x591, 0x5bd; 0x5bf, 0x5bf; 0x5c1, 0x5c2; 0x5c4, 0x5c5; 0x5c7, 0x5c9; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x610, 0x61a; 0x620, 0x669; 0x66e, 0x6d3; 0x6d5, 0x6dc; 0x6df, 0x6e8; 0x6ea, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x74a; 0x74d, 0x7b1; 0x7c0, 0x7f5; 0x7fa, 0x7fa; 0x7fd, 0x7fd; 0x800, 0x82d; 0x840, 0x85b; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x897, 0x8e1; 0x8e3, 0x963; 0x966, 0x96f; 0x971, 0x983; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bc, 0x9c4; 0x9c7, 0x9c8; 0x9cb, 0x9ce; 0x9d7, 0x9d7; 0x9dc, 0x9dd; 0x9df, 0x9e3; 0x9e6, 0x9f1; 0x9fc, 0x9fc; 0x9fe, 0x9fe; 0xa01, 0xa03; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa3c, 0xa3c; 0xa3e, 0xa42; 0xa47, 0xa48; 0xa4b, 0xa4d; 0xa51, 0xa51; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa66, 0xa75; 0xa81, 0xa83; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabc, 0xac5; 0xac7, 0xac9; 0xacb, 0xacd; 0xad0, 0xad0; 0xae0, 0xae3; 0xae6, 0xaef; 0xaf9, 0xaff; 0xb01, 0xb03; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3c, 0xb44; 0xb47, 0xb48; 0xb4b, 0xb4d; 0xb53, 0xb57; 0xb5c, 0xb5d; 0xb5f, 0xb63; 0xb66, 0xb6f; 0xb71, 0xb71; 0xb82, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbbe, 0xbc2; 0xbc6, 0xbc8; 0xbca, 0xbcd; 0xbd0, 0xbd0; 0xbd7, 0xbd7; 0xbe6, 0xbef; 0xc00, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3c, 0xc44; 0xc46, 0xc48; 0xc4a, 0xc4d; 0xc55, 0xc56; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc63; 0xc66, 0xc6f; 0xc80, 0xc83; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbc, 0xcc4; 0xcc6, 0xcc8; 0xcca, 0xccd; 0xcd5, 0xcd6; 0xcdc, 0xcde; 0xce0, 0xce3; 0xce6, 0xcef; 0xcf1, 0xcf3; 0xd00, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd44; 0xd46, 0xd48; 0xd4a, 0xd4e; 0xd54, 0xd57; 0xd5f, 0xd63; 0xd66, 0xd6f; 0xd7a, 0xd7f; 0xd81, 0xd83; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xdca, 0xdca; 0xdcf, 0xdd4; 0xdd6, 0xdd6; 0xdd8, 0xddf; 0xde6, 0xdef; 0xdf2, 0xdf3; 0xe01, 0xe3a; 0xe40, 0xe4e; 0xe50, 0xe59; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xec8, 0xece; 0xed0, 0xed9; 0xedc, 0xedf; 0xf00, 0xf00; 0xf18, 0xf19; 0xf20, 0xf29; 0xf35, 0xf35; 0xf37, 0xf37; 0xf39, 0xf39; 0xf3e, 0xf47; 0xf49, 0xf6c; 0xf71, 0xf84; 0xf86, 0xf97; 0xf99, 0xfbc; 0xfc6, 0xfc6; 0x1000, 0x1049; 0x1050, 0x109d; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x135d, 0x135f; 0x1369, 0x1371; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1715; 0x171f, 0x1734; 0x1740, 0x1753; 0x1760, 0x176c; 0x176e, 0x1770; 0x1772, 0x1773; 0x1780, 0x17d3; 0x17d7, 0x17d7; 0x17dc, 0x17dd; 0x17e0, 0x17e9; 0x180b, 0x180d; 0x180f, 0x1819; 0x1820, 0x1878; 0x1880, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1920, 0x192b; 0x1930, 0x193b; 0x1946, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x19d0, 0x19da; 0x1a00, 0x1a1b; 0x1a20, 0x1a5e; 0x1a60, 0x1a7c; 0x1a7f, 0x1a89; 0x1a90, 0x1a99; 0x1aa7, 0x1aa7; 0x1ab0, 0x1abd; 0x1abf, 0x1af0; 0x1b00, 0x1b4c; 0x1b50, 0x1b59; 0x1b6b, 0x1b73; 0x1b80, 0x1bf3; 0x1c00, 0x1c37; 0x1c40, 0x1c49; 0x1c4d, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1cd0, 0x1cd2; 0x1cd4, 0x1cfa; 0x1d00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x200c, 0x200d; 0x203f, 0x2040; 0x2054, 0x2054; 0x2071, 0x2071; 0x207f, 0x207f; 0x208f, 0x209f; 0x20d0, 0x20dc; 0x20e1, 0x20e1; 0x20e5, 0x20f0; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d7f, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x2de0, 0x2dff; 0x3005, 0x3007; 0x3021, 0x302f; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x3099, 0x309a; 0x309d, 0x309f; 0x30a1, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa62b; 0xa640, 0xa66f; 0xa674, 0xa67d; 0xa67f, 0xa6f1; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f1, 0xa827; 0xa82c, 0xa82c; 0xa840, 0xa873; 0xa880, 0xa8c5; 0xa8d0, 0xa8d9; 0xa8e0, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa92d; 0xa930, 0xa953; 0xa960, 0xa97c; 0xa980, 0xa9c0; 0xa9cf, 0xa9d9; 0xa9e0, 0xa9fe; 0xaa00, 0xaa36; 0xaa40, 0xaa4d; 0xaa50, 0xaa59; 0xaa60, 0xaa76; 0xaa7a, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaef; 0xaaf2, 0xaaf6; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab6c, 0xab6d; 0xab70, 0xabea; 0xabec, 0xabed; 0xabf0, 0xabf9; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfc5d; 0xfc64, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdf9; 0xfe00, 0xfe0f; 0xfe20, 0xfe2f; 0xfe33, 0xfe34; 0xfe4d, 0xfe4f; 0xfe71, 0xfe71; 0xfe73, 0xfe73; 0xfe77, 0xfe77; 0xfe79, 0xfe79; 0xfe7b, 0xfe7b; 0xfe7d, 0xfe7d; 0xfe7f, 0xfefc; 0xff10, 0xff19; 0xff21, 0xff3a; 0xff3f, 0xff3f; 0xff41, 0xff5a; 0xff65, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x101fd, 0x101fd; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x102e0, 0x102e0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x1037a; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104a0, 0x104a9; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a03; 0x10a05, 0x10a06; 0x10a0c, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a38, 0x10a3a; 0x10a3f, 0x10a3f; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae6; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d27; 0x10d30, 0x10d39; 0x10d40, 0x10d65; 0x10d69, 0x10d6d; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eab, 0x10eac; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10ecb, 0x10ecf; 0x10ed9, 0x10eee; 0x10ef0, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f50; 0x10f70, 0x10f85; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11000, 0x11046; 0x11066, 0x11075; 0x1107f, 0x110ba; 0x110c2, 0x110c2; 0x110d0, 0x110e8; 0x110f0, 0x110f9; 0x11100, 0x11134; 0x11136, 0x1113f; 0x11144, 0x11147; 0x11150, 0x11173; 0x11176, 0x11176; 0x11180, 0x111c4; 0x111c9, 0x111cc; 0x111ce, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x11237; 0x1123e, 0x11241; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112ea; 0x112f0, 0x112f9; 0x11300, 0x11303; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133b, 0x11344; 0x11347, 0x11348; 0x1134b, 0x1134d; 0x11350, 0x11350; 0x11357, 0x11357; 0x1135d, 0x11363; 0x11366, 0x1136c; 0x11370, 0x11374; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113c0; 0x113c2, 0x113c2; 0x113c5, 0x113c5; 0x113c7, 0x113ca; 0x113cc, 0x113d3; 0x113e1, 0x113e2; 0x11400, 0x1144a; 0x11450, 0x11459; 0x1145e, 0x11461; 0x11480, 0x114c5; 0x114c7, 0x114c7; 0x114d0, 0x114d9; 0x11580, 0x115b5; 0x115b8, 0x115c0; 0x115d8, 0x115dd; 0x11600, 0x11640; 0x11644, 0x11644; 0x11650, 0x11659; 0x11680, 0x116b8; 0x116c0, 0x116c9; 0x116d0, 0x116e3; 0x11700, 0x1171a; 0x1171d, 0x1172b; 0x11730, 0x11739; 0x11740, 0x11746; 0x11800, 0x1183a; 0x118a0, 0x118e9; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x11935; 0x11937, 0x11938; 0x1193b, 0x11943; 0x11950, 0x11959; 0x119a0, 0x119a7; 0x119aa, 0x119d7; 0x119da, 0x119e1; 0x119e3, 0x119e4; 0x11a00, 0x11a3e; 0x11a47, 0x11a47; 0x11a50, 0x11a99; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b0a, 0x11b0a; 0x11b60, 0x11b67; 0x11bc0, 0x11be0; 0x11bf0, 0x11bf9; 0x11c00, 0x11c08; 0x11c0a, 0x11c36; 0x11c38, 0x11c40; 0x11c50, 0x11c59; 0x11c72, 0x11c8f; 0x11c92, 0x11ca7; 0x11ca9, 0x11cb6; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d36; 0x11d3a, 0x11d3a; 0x11d3c, 0x11d3d; 0x11d3f, 0x11d47; 0x11d50, 0x11d59; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d8e; 0x11d90, 0x11d91; 0x11d93, 0x11d98; 0x11da0, 0x11da9; 0x11db0, 0x11ddb; 0x11de0, 0x11de9; 0x11df0, 0x11df1; 0x11ee0, 0x11ef6; 0x11f00, 0x11f10; 0x11f12, 0x11f3a; 0x11f3e, 0x11f42; 0x11f50, 0x11f5a; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246f; 0x12475, 0x12543; 0x12550, 0x12686; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13440, 0x13455; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x16139; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a60, 0x16a69; 0x16a70, 0x16abe; 0x16ac0, 0x16ac9; 0x16ad0, 0x16aed; 0x16af0, 0x16af4; 0x16b00, 0x16b36; 0x16b40, 0x16b43; 0x16b50, 0x16b59; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16d70, 0x16d79; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f4f, 0x16f87; 0x16f8f, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe4; 0x16ff0, 0x16ff6; 0x17000, 0x18cda; 0x18cff, 0x18d20; 0x18d80, 0x18df2; 0x18e00, 0x19191; 0x191a0, 0x191d2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b128; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b168; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1bc9d, 0x1bc9e; 0x1ccf0, 0x1ccf9; 0x1cf00, 0x1cf2d; 0x1cf30, 0x1cf46; 0x1d127, 0x1d128; 0x1d165, 0x1d169; 0x1d16d, 0x1d172; 0x1d17b, 0x1d182; 0x1d185, 0x1d18b; 0x1d1aa, 0x1d1ad; 0x1d242, 0x1d244; 0x1d250, 0x1d252; 0x1d25b, 0x1d25c; 0x1d25f, 0x1d25f; 0x1d280, 0x1d281; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1d7ce, 0x1d7ff; 0x1da00, 0x1da36; 0x1da3b, 0x1da6c; 0x1da75, 0x1da75; 0x1da84, 0x1da84; 0x1da9b, 0x1da9f; 0x1daa1, 0x1daaf; 0x1df00, 0x1df81; 0x1df90, 0x1df96; 0x1dfcd, 0x1e006; 0x1e008, 0x1e018; 0x1e01b, 0x1e021; 0x1e023, 0x1e024; 0x1e026, 0x1e02a; 0x1e030, 0x1e06d; 0x1e08f, 0x1e08f; 0x1e100, 0x1e12c; 0x1e130, 0x1e13d; 0x1e140, 0x1e149; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ae; 0x1e2c0, 0x1e2f9; 0x1e4d0, 0x1e4f9; 0x1e5d0, 0x1e5fa; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6f5; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e8d0, 0x1e8d6; 0x1e900, 0x1e94b; 0x1e950, 0x1e959; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x1fbf0, 0x1fbf9; 0x20000, 0x2a6df; 0x2a700, 0x2b81e; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0x3d000, 0x3fc3f; 0xe0100, 0xe01ef] let xid_start = Cset.of_list [0x41, 0x5a; 0x61, 0x7a; 0xaa, 0xaa; 0xb5, 0xb5; 0xba, 0xba; 0xc0, 0xd6; 0xd8, 0xf6; 0xf8, 0x2c1; 0x2c6, 0x2d1; 0x2e0, 0x2e4; 0x2ec, 0x2ec; 0x2ee, 0x2ee; 0x370, 0x374; 0x376, 0x377; 0x37b, 0x37d; 0x37f, 0x37f; 0x386, 0x386; 0x388, 0x38a; 0x38c, 0x38c; 0x38e, 0x3a1; 0x3a3, 0x3f5; 0x3f7, 0x481; 0x48a, 0x52f; 0x531, 0x556; 0x558, 0x559; 0x560, 0x588; 0x58b, 0x58c; 0x5d0, 0x5ea; 0x5ef, 0x5f2; 0x620, 0x64a; 0x66e, 0x66f; 0x671, 0x6d3; 0x6d5, 0x6d5; 0x6e5, 0x6e6; 0x6ee, 0x6ef; 0x6fa, 0x6fc; 0x6ff, 0x6ff; 0x710, 0x710; 0x712, 0x72f; 0x74d, 0x7a5; 0x7b1, 0x7b1; 0x7ca, 0x7ea; 0x7f4, 0x7f5; 0x7fa, 0x7fa; 0x800, 0x815; 0x81a, 0x81a; 0x824, 0x824; 0x828, 0x828; 0x840, 0x858; 0x860, 0x86a; 0x870, 0x887; 0x889, 0x88f; 0x8a0, 0x8c9; 0x904, 0x939; 0x93d, 0x93d; 0x950, 0x950; 0x958, 0x961; 0x971, 0x980; 0x985, 0x98c; 0x98f, 0x990; 0x993, 0x9a8; 0x9aa, 0x9b0; 0x9b2, 0x9b2; 0x9b6, 0x9b9; 0x9bd, 0x9bd; 0x9ce, 0x9ce; 0x9dc, 0x9dd; 0x9df, 0x9e1; 0x9f0, 0x9f1; 0x9fc, 0x9fc; 0xa05, 0xa0a; 0xa0f, 0xa10; 0xa13, 0xa28; 0xa2a, 0xa30; 0xa32, 0xa33; 0xa35, 0xa36; 0xa38, 0xa39; 0xa59, 0xa5c; 0xa5e, 0xa5e; 0xa72, 0xa74; 0xa85, 0xa8d; 0xa8f, 0xa91; 0xa93, 0xaa8; 0xaaa, 0xab0; 0xab2, 0xab3; 0xab5, 0xab9; 0xabd, 0xabd; 0xad0, 0xad0; 0xae0, 0xae1; 0xaf9, 0xaf9; 0xb05, 0xb0c; 0xb0f, 0xb10; 0xb13, 0xb28; 0xb2a, 0xb30; 0xb32, 0xb33; 0xb35, 0xb39; 0xb3d, 0xb3d; 0xb5c, 0xb5d; 0xb5f, 0xb61; 0xb71, 0xb71; 0xb83, 0xb83; 0xb85, 0xb8a; 0xb8e, 0xb90; 0xb92, 0xb95; 0xb99, 0xb9a; 0xb9c, 0xb9c; 0xb9e, 0xb9f; 0xba3, 0xba4; 0xba8, 0xbaa; 0xbae, 0xbb9; 0xbd0, 0xbd0; 0xc05, 0xc0c; 0xc0e, 0xc10; 0xc12, 0xc28; 0xc2a, 0xc39; 0xc3d, 0xc3d; 0xc58, 0xc5a; 0xc5c, 0xc5d; 0xc60, 0xc61; 0xc80, 0xc80; 0xc85, 0xc8c; 0xc8e, 0xc90; 0xc92, 0xca8; 0xcaa, 0xcb3; 0xcb5, 0xcb9; 0xcbd, 0xcbd; 0xcdc, 0xcde; 0xce0, 0xce1; 0xcf1, 0xcf2; 0xd04, 0xd0c; 0xd0e, 0xd10; 0xd12, 0xd3a; 0xd3d, 0xd3d; 0xd4e, 0xd4e; 0xd54, 0xd56; 0xd5f, 0xd61; 0xd7a, 0xd7f; 0xd85, 0xd96; 0xd9a, 0xdb1; 0xdb3, 0xdbb; 0xdbd, 0xdbd; 0xdc0, 0xdc6; 0xe01, 0xe30; 0xe32, 0xe32; 0xe40, 0xe46; 0xe81, 0xe82; 0xe84, 0xe84; 0xe86, 0xe8a; 0xe8c, 0xea3; 0xea5, 0xea5; 0xea7, 0xeb0; 0xeb2, 0xeb2; 0xebd, 0xebd; 0xec0, 0xec4; 0xec6, 0xec6; 0xedc, 0xedf; 0xf00, 0xf00; 0xf40, 0xf47; 0xf49, 0xf6c; 0xf88, 0xf8c; 0x1000, 0x102a; 0x103f, 0x103f; 0x1050, 0x1055; 0x105a, 0x105d; 0x1061, 0x1061; 0x1065, 0x1066; 0x106e, 0x1070; 0x1075, 0x1081; 0x108e, 0x108e; 0x10a0, 0x10c5; 0x10c7, 0x10c7; 0x10cd, 0x10cd; 0x10d0, 0x10fa; 0x10fc, 0x1248; 0x124a, 0x124d; 0x1250, 0x1256; 0x1258, 0x1258; 0x125a, 0x125d; 0x1260, 0x1288; 0x128a, 0x128d; 0x1290, 0x12b0; 0x12b2, 0x12b5; 0x12b8, 0x12be; 0x12c0, 0x12c0; 0x12c2, 0x12c5; 0x12c8, 0x12d6; 0x12d8, 0x1310; 0x1312, 0x1315; 0x1318, 0x135a; 0x1380, 0x138f; 0x13a0, 0x13f5; 0x13f8, 0x13fd; 0x1401, 0x166c; 0x166f, 0x167f; 0x1681, 0x169a; 0x16a0, 0x16ea; 0x16ee, 0x16f8; 0x1700, 0x1711; 0x171f, 0x1731; 0x1740, 0x1751; 0x1760, 0x176c; 0x176e, 0x1770; 0x1780, 0x17b3; 0x17d7, 0x17d7; 0x17dc, 0x17dc; 0x1820, 0x1878; 0x1880, 0x18a8; 0x18aa, 0x18aa; 0x18b0, 0x18f5; 0x1900, 0x191e; 0x1950, 0x196d; 0x1970, 0x1974; 0x1980, 0x19ab; 0x19b0, 0x19c9; 0x1a00, 0x1a16; 0x1a20, 0x1a54; 0x1aa7, 0x1aa7; 0x1b05, 0x1b33; 0x1b45, 0x1b4c; 0x1b83, 0x1ba0; 0x1bae, 0x1baf; 0x1bba, 0x1be5; 0x1c00, 0x1c23; 0x1c4d, 0x1c4f; 0x1c5a, 0x1c7d; 0x1c80, 0x1c8a; 0x1c90, 0x1cba; 0x1cbd, 0x1cbf; 0x1ce9, 0x1cec; 0x1cee, 0x1cf3; 0x1cf5, 0x1cf6; 0x1cfa, 0x1cfa; 0x1d00, 0x1dbf; 0x1e00, 0x1f15; 0x1f18, 0x1f1d; 0x1f20, 0x1f45; 0x1f48, 0x1f4d; 0x1f50, 0x1f57; 0x1f59, 0x1f59; 0x1f5b, 0x1f5b; 0x1f5d, 0x1f5d; 0x1f5f, 0x1f7d; 0x1f80, 0x1fb4; 0x1fb6, 0x1fbc; 0x1fbe, 0x1fbe; 0x1fc2, 0x1fc4; 0x1fc6, 0x1fcc; 0x1fd0, 0x1fd3; 0x1fd6, 0x1fdb; 0x1fe0, 0x1fec; 0x1ff2, 0x1ff4; 0x1ff6, 0x1ffc; 0x2071, 0x2071; 0x207f, 0x207f; 0x208f, 0x209f; 0x2102, 0x2102; 0x2107, 0x2107; 0x210a, 0x2113; 0x2115, 0x2115; 0x2118, 0x211d; 0x2124, 0x2124; 0x2126, 0x2126; 0x2128, 0x2128; 0x212a, 0x2139; 0x213c, 0x213f; 0x2145, 0x2149; 0x214e, 0x214e; 0x2160, 0x2188; 0x2c00, 0x2ce4; 0x2ceb, 0x2cee; 0x2cf2, 0x2cf3; 0x2d00, 0x2d25; 0x2d27, 0x2d27; 0x2d2d, 0x2d2d; 0x2d30, 0x2d67; 0x2d6f, 0x2d6f; 0x2d80, 0x2d96; 0x2da0, 0x2da6; 0x2da8, 0x2dae; 0x2db0, 0x2db6; 0x2db8, 0x2dbe; 0x2dc0, 0x2dc6; 0x2dc8, 0x2dce; 0x2dd0, 0x2dd6; 0x2dd8, 0x2dde; 0x3005, 0x3007; 0x3021, 0x3029; 0x3031, 0x3035; 0x3038, 0x303c; 0x3041, 0x3096; 0x309d, 0x309f; 0x30a1, 0x30fa; 0x30fc, 0x30ff; 0x3105, 0x312f; 0x3131, 0x318e; 0x31a0, 0x31bf; 0x31f0, 0x31ff; 0x3400, 0x4dbf; 0x4e00, 0xa48c; 0xa4d0, 0xa4fd; 0xa500, 0xa60c; 0xa610, 0xa61f; 0xa62a, 0xa62b; 0xa640, 0xa66e; 0xa67f, 0xa69d; 0xa6a0, 0xa6ef; 0xa717, 0xa71f; 0xa722, 0xa788; 0xa78b, 0xa7dd; 0xa7e2, 0xa7e2; 0xa7f1, 0xa801; 0xa803, 0xa805; 0xa807, 0xa80a; 0xa80c, 0xa822; 0xa840, 0xa873; 0xa882, 0xa8b3; 0xa8f2, 0xa8f7; 0xa8fb, 0xa8fb; 0xa8fd, 0xa8fe; 0xa90a, 0xa925; 0xa930, 0xa946; 0xa960, 0xa97c; 0xa984, 0xa9b2; 0xa9cf, 0xa9cf; 0xa9e0, 0xa9e4; 0xa9e6, 0xa9ef; 0xa9fa, 0xa9fe; 0xaa00, 0xaa28; 0xaa40, 0xaa42; 0xaa44, 0xaa4b; 0xaa60, 0xaa76; 0xaa7a, 0xaa7a; 0xaa7e, 0xaaaf; 0xaab1, 0xaab1; 0xaab5, 0xaab6; 0xaab9, 0xaabd; 0xaac0, 0xaac0; 0xaac2, 0xaac2; 0xaadb, 0xaadd; 0xaae0, 0xaaea; 0xaaf2, 0xaaf4; 0xab01, 0xab06; 0xab09, 0xab0e; 0xab11, 0xab16; 0xab20, 0xab26; 0xab28, 0xab2e; 0xab30, 0xab5a; 0xab5c, 0xab69; 0xab6c, 0xab6d; 0xab70, 0xabe2; 0xac00, 0xd7a3; 0xd7b0, 0xd7c6; 0xd7cb, 0xd7fb; 0xf900, 0xfa6d; 0xfa70, 0xfad9; 0xfb00, 0xfb06; 0xfb13, 0xfb17; 0xfb1d, 0xfb1d; 0xfb1f, 0xfb28; 0xfb2a, 0xfb36; 0xfb38, 0xfb3c; 0xfb3e, 0xfb3e; 0xfb40, 0xfb41; 0xfb43, 0xfb44; 0xfb46, 0xfbb1; 0xfbd3, 0xfc5d; 0xfc64, 0xfd3d; 0xfd50, 0xfd8f; 0xfd92, 0xfdc7; 0xfdf0, 0xfdf9; 0xfe71, 0xfe71; 0xfe73, 0xfe73; 0xfe77, 0xfe77; 0xfe79, 0xfe79; 0xfe7b, 0xfe7b; 0xfe7d, 0xfe7d; 0xfe7f, 0xfefc; 0xff21, 0xff3a; 0xff41, 0xff5a; 0xff66, 0xff9d; 0xffa0, 0xffbe; 0xffc2, 0xffc7; 0xffca, 0xffcf; 0xffd2, 0xffd7; 0xffda, 0xffdc; 0x10000, 0x1000b; 0x1000d, 0x10026; 0x10028, 0x1003a; 0x1003c, 0x1003d; 0x1003f, 0x1004d; 0x10050, 0x1005d; 0x10080, 0x100fa; 0x10140, 0x10174; 0x10280, 0x1029c; 0x102a0, 0x102d0; 0x10300, 0x1031f; 0x1032d, 0x1034a; 0x10350, 0x10375; 0x10380, 0x1039d; 0x103a0, 0x103c3; 0x103c8, 0x103cf; 0x103d1, 0x103d5; 0x10400, 0x1049d; 0x104b0, 0x104d3; 0x104d8, 0x104fb; 0x10500, 0x10527; 0x10530, 0x10563; 0x10570, 0x1057a; 0x1057c, 0x1058a; 0x1058c, 0x10592; 0x10594, 0x10595; 0x10597, 0x105a1; 0x105a3, 0x105b1; 0x105b3, 0x105b9; 0x105bb, 0x105bc; 0x105c0, 0x105f3; 0x10600, 0x10736; 0x10740, 0x10755; 0x10760, 0x10767; 0x10780, 0x10785; 0x10787, 0x107b0; 0x107b2, 0x107bf; 0x10800, 0x10805; 0x10808, 0x10808; 0x1080a, 0x10835; 0x10837, 0x10838; 0x1083c, 0x1083c; 0x1083f, 0x10855; 0x10860, 0x10876; 0x10880, 0x1089e; 0x108e0, 0x108f2; 0x108f4, 0x108f5; 0x10900, 0x10915; 0x10920, 0x10939; 0x10940, 0x10959; 0x10980, 0x109b7; 0x109be, 0x109bf; 0x10a00, 0x10a00; 0x10a10, 0x10a13; 0x10a15, 0x10a17; 0x10a19, 0x10a35; 0x10a60, 0x10a7c; 0x10a80, 0x10a9c; 0x10ac0, 0x10ac7; 0x10ac9, 0x10ae4; 0x10b00, 0x10b35; 0x10b40, 0x10b55; 0x10b60, 0x10b72; 0x10b80, 0x10b91; 0x10c00, 0x10c48; 0x10c80, 0x10cb2; 0x10cc0, 0x10cf2; 0x10d00, 0x10d23; 0x10d4a, 0x10d65; 0x10d6f, 0x10d85; 0x10e80, 0x10ea9; 0x10eb0, 0x10eb1; 0x10ec2, 0x10ec7; 0x10ed9, 0x10eee; 0x10f00, 0x10f1c; 0x10f27, 0x10f27; 0x10f30, 0x10f45; 0x10f70, 0x10f81; 0x10fb0, 0x10fc4; 0x10fe0, 0x10ff6; 0x11003, 0x11037; 0x11071, 0x11072; 0x11075, 0x11075; 0x11083, 0x110af; 0x110d0, 0x110e8; 0x11103, 0x11126; 0x11144, 0x11144; 0x11147, 0x11147; 0x11150, 0x11172; 0x11176, 0x11176; 0x11183, 0x111b2; 0x111c1, 0x111c4; 0x111da, 0x111da; 0x111dc, 0x111dc; 0x11200, 0x11211; 0x11213, 0x1122b; 0x1123f, 0x11240; 0x11280, 0x11286; 0x11288, 0x11288; 0x1128a, 0x1128d; 0x1128f, 0x1129d; 0x1129f, 0x112a8; 0x112b0, 0x112de; 0x11305, 0x1130c; 0x1130f, 0x11310; 0x11313, 0x11328; 0x1132a, 0x11330; 0x11332, 0x11333; 0x11335, 0x11339; 0x1133d, 0x1133d; 0x11350, 0x11350; 0x1135d, 0x11361; 0x11380, 0x11389; 0x1138b, 0x1138b; 0x1138e, 0x1138e; 0x11390, 0x113b5; 0x113b7, 0x113b7; 0x113d1, 0x113d1; 0x113d3, 0x113d3; 0x11400, 0x11434; 0x11447, 0x1144a; 0x1145f, 0x11461; 0x11480, 0x114af; 0x114c4, 0x114c5; 0x114c7, 0x114c7; 0x11580, 0x115ae; 0x115d8, 0x115db; 0x11600, 0x1162f; 0x11644, 0x11644; 0x11680, 0x116aa; 0x116b8, 0x116b8; 0x11700, 0x1171a; 0x11740, 0x11746; 0x11800, 0x1182b; 0x118a0, 0x118df; 0x118ff, 0x11906; 0x11909, 0x11909; 0x1190c, 0x11913; 0x11915, 0x11916; 0x11918, 0x1192f; 0x1193f, 0x1193f; 0x11941, 0x11941; 0x119a0, 0x119a7; 0x119aa, 0x119d0; 0x119e1, 0x119e1; 0x119e3, 0x119e3; 0x11a00, 0x11a00; 0x11a0b, 0x11a32; 0x11a3a, 0x11a3a; 0x11a50, 0x11a50; 0x11a5c, 0x11a89; 0x11a9d, 0x11a9d; 0x11ab0, 0x11af8; 0x11b0a, 0x11b0a; 0x11bc0, 0x11be0; 0x11c00, 0x11c08; 0x11c0a, 0x11c2e; 0x11c40, 0x11c40; 0x11c72, 0x11c8f; 0x11d00, 0x11d06; 0x11d08, 0x11d09; 0x11d0b, 0x11d30; 0x11d46, 0x11d46; 0x11d60, 0x11d65; 0x11d67, 0x11d68; 0x11d6a, 0x11d89; 0x11d98, 0x11d98; 0x11db0, 0x11ddb; 0x11df1, 0x11df1; 0x11ee0, 0x11ef2; 0x11f02, 0x11f02; 0x11f04, 0x11f10; 0x11f12, 0x11f33; 0x11fb0, 0x11fb0; 0x12000, 0x12399; 0x12400, 0x1246f; 0x12475, 0x12543; 0x12550, 0x12686; 0x12f90, 0x12ff0; 0x13000, 0x1342f; 0x13441, 0x13446; 0x13460, 0x143fa; 0x14400, 0x14646; 0x16100, 0x1611d; 0x16800, 0x16a38; 0x16a40, 0x16a5e; 0x16a70, 0x16abe; 0x16ad0, 0x16aed; 0x16b00, 0x16b2f; 0x16b40, 0x16b43; 0x16b63, 0x16b77; 0x16b7d, 0x16b8f; 0x16d40, 0x16d6c; 0x16e40, 0x16e7f; 0x16ea0, 0x16eb8; 0x16ebb, 0x16ed3; 0x16f00, 0x16f4a; 0x16f50, 0x16f50; 0x16f93, 0x16f9f; 0x16fe0, 0x16fe1; 0x16fe3, 0x16fe3; 0x16ff2, 0x16ff6; 0x17000, 0x18cda; 0x18cff, 0x18d20; 0x18d80, 0x18df2; 0x18e00, 0x19191; 0x191a0, 0x191d2; 0x1aff0, 0x1aff3; 0x1aff5, 0x1affb; 0x1affd, 0x1affe; 0x1b000, 0x1b128; 0x1b132, 0x1b132; 0x1b150, 0x1b152; 0x1b155, 0x1b155; 0x1b164, 0x1b168; 0x1b170, 0x1b2fb; 0x1bc00, 0x1bc6a; 0x1bc70, 0x1bc7c; 0x1bc80, 0x1bc88; 0x1bc90, 0x1bc99; 0x1d400, 0x1d454; 0x1d456, 0x1d49c; 0x1d49e, 0x1d49f; 0x1d4a2, 0x1d4a2; 0x1d4a5, 0x1d4a6; 0x1d4a9, 0x1d4ac; 0x1d4ae, 0x1d4b9; 0x1d4bb, 0x1d4bb; 0x1d4bd, 0x1d4c3; 0x1d4c5, 0x1d505; 0x1d507, 0x1d50a; 0x1d50d, 0x1d514; 0x1d516, 0x1d51c; 0x1d51e, 0x1d539; 0x1d53b, 0x1d53e; 0x1d540, 0x1d544; 0x1d546, 0x1d546; 0x1d54a, 0x1d550; 0x1d552, 0x1d6a6; 0x1d6a8, 0x1d6c0; 0x1d6c2, 0x1d6da; 0x1d6dc, 0x1d6fa; 0x1d6fc, 0x1d714; 0x1d716, 0x1d734; 0x1d736, 0x1d74e; 0x1d750, 0x1d76e; 0x1d770, 0x1d788; 0x1d78a, 0x1d7a8; 0x1d7aa, 0x1d7c2; 0x1d7c4, 0x1d7cb; 0x1df00, 0x1df81; 0x1df90, 0x1df96; 0x1dfcd, 0x1dfff; 0x1e030, 0x1e06d; 0x1e100, 0x1e12c; 0x1e137, 0x1e13d; 0x1e14e, 0x1e14e; 0x1e290, 0x1e2ad; 0x1e2c0, 0x1e2eb; 0x1e4d0, 0x1e4eb; 0x1e5d0, 0x1e5ed; 0x1e5f0, 0x1e5f0; 0x1e6c0, 0x1e6de; 0x1e6e0, 0x1e6e2; 0x1e6e4, 0x1e6e5; 0x1e6e7, 0x1e6ed; 0x1e6f0, 0x1e6f4; 0x1e6fe, 0x1e6ff; 0x1e7e0, 0x1e7e6; 0x1e7e8, 0x1e7eb; 0x1e7ed, 0x1e7ee; 0x1e7f0, 0x1e7fe; 0x1e800, 0x1e8c4; 0x1e900, 0x1e943; 0x1e94b, 0x1e94b; 0x1ee00, 0x1ee03; 0x1ee05, 0x1ee1f; 0x1ee21, 0x1ee22; 0x1ee24, 0x1ee24; 0x1ee27, 0x1ee27; 0x1ee29, 0x1ee32; 0x1ee34, 0x1ee37; 0x1ee39, 0x1ee39; 0x1ee3b, 0x1ee3b; 0x1ee42, 0x1ee42; 0x1ee47, 0x1ee47; 0x1ee49, 0x1ee49; 0x1ee4b, 0x1ee4b; 0x1ee4d, 0x1ee4f; 0x1ee51, 0x1ee52; 0x1ee54, 0x1ee54; 0x1ee57, 0x1ee57; 0x1ee59, 0x1ee59; 0x1ee5b, 0x1ee5b; 0x1ee5d, 0x1ee5d; 0x1ee5f, 0x1ee5f; 0x1ee61, 0x1ee62; 0x1ee64, 0x1ee64; 0x1ee67, 0x1ee6a; 0x1ee6c, 0x1ee72; 0x1ee74, 0x1ee77; 0x1ee79, 0x1ee7c; 0x1ee7e, 0x1ee7e; 0x1ee80, 0x1ee89; 0x1ee8b, 0x1ee9b; 0x1eea1, 0x1eea3; 0x1eea5, 0x1eea9; 0x1eeab, 0x1eebb; 0x20000, 0x2a6df; 0x2a700, 0x2b81e; 0x2b820, 0x2cead; 0x2ceb0, 0x2ebe0; 0x2ebf0, 0x2ee5d; 0x2f800, 0x2fa1d; 0x30000, 0x3134a; 0x31350, 0x33479; 0x3d000, 0x3fc3f] let list = [ ("alphabetic", alphabetic); ("ascii_hex_digit", ascii_hex_digit); ("hex_digit", hex_digit); ("id_continue", id_continue); ("id_start", id_start); ("lowercase", lowercase); ("math", math); ("other_alphabetic", other_alphabetic); ("other_lowercase", other_lowercase); ("other_math", other_math); ("other_uppercase", other_uppercase); ("uppercase", uppercase); ("white_space", white_space); ("xid_continue", xid_continue); ("xid_start", xid_start) ] (* ignoring: - bidi_control - case_ignorable - cased - changes_when_casefolded - changes_when_casemapped - changes_when_lowercased - changes_when_titlecased - changes_when_uppercased - dash - default_ignorable_code_point - deprecated - diacritic - extender - grapheme_base - grapheme_extend - grapheme_link - hyphen - id_compat_math_continue - id_compat_math_start - ideographic - ids_binary_operator - ids_trinary_operator - ids_unary_operator - join_control - logical_order_exception - modifier_combining_mark - noncharacter_code_point - other_default_ignorable_code_point - other_grapheme_extend - other_id_continue - other_id_start - pattern_syntax - pattern_white_space - prepended_concatenation_mark - quotation_mark - radical - regional_indicator - sentence_terminal - soft_dotted - terminal_punctuation - unified_ideograph - variation_selector *) end sedlex-3.8.1/src/syntax/unicode.mli000066400000000000000000000022601526113672700172670ustar00rootroot00000000000000(* Version is automatically generated from unicode data at * src/generator/data *) open Sedlex_compiler val version : string module Categories : sig val cc : Cset.t val cf : Cset.t val cn : Cset.t val co : Cset.t val cs : Cset.t val ll : Cset.t val lm : Cset.t val lo : Cset.t val lt : Cset.t val lu : Cset.t val mc : Cset.t val me : Cset.t val mn : Cset.t val nd : Cset.t val nl : Cset.t val no : Cset.t val pc : Cset.t val pd : Cset.t val pe : Cset.t val pf : Cset.t val pi : Cset.t val po : Cset.t val ps : Cset.t val sc : Cset.t val sk : Cset.t val sm : Cset.t val so : Cset.t val zl : Cset.t val zp : Cset.t val zs : Cset.t val list : (string * Cset.t) list end module Properties : sig val alphabetic : Cset.t val ascii_hex_digit : Cset.t val hex_digit : Cset.t val id_continue : Cset.t val id_start : Cset.t val lowercase : Cset.t val math : Cset.t val other_alphabetic : Cset.t val other_lowercase : Cset.t val other_math : Cset.t val other_uppercase : Cset.t val uppercase : Cset.t val white_space : Cset.t val xid_continue : Cset.t val xid_start : Cset.t val list : (string * Cset.t) list end sedlex-3.8.1/src/syntax/utf8.ml000066400000000000000000000031641526113672700163620ustar00rootroot00000000000000open Sedlexing let unsafe_byte s j = Char.code (String.unsafe_get s j) let malformed s j l = `Malformed (String.sub s j l) let r_utf_8 s j l = (* assert (0 <= j && 0 <= l && j + l <= String.length s); *) let uchar c = `Uchar (Uchar.unsafe_of_int c) in match l with | 1 -> uchar (unsafe_byte s j) | 2 -> ( let b0 = unsafe_byte s j in let b1 = unsafe_byte s (j + 1) in match Utf8.Helper.check_two b0 b1 with | i -> uchar i | exception MalFormed -> malformed s j l) | 3 -> ( let b0 = unsafe_byte s j in let b1 = unsafe_byte s (j + 1) in let b2 = unsafe_byte s (j + 2) in match Utf8.Helper.check_three b0 b1 b2 with | i -> uchar i | exception MalFormed -> malformed s j l) | 4 -> ( let b0 = unsafe_byte s j in let b1 = unsafe_byte s (j + 1) in let b2 = unsafe_byte s (j + 2) in let b3 = unsafe_byte s (j + 3) in match Utf8.Helper.check_four b0 b1 b2 b3 with | i -> uchar i | exception MalFormed -> malformed s j l) | _ -> assert false let fold ~f acc s = let rec loop acc f s i last = if i > last then acc else ( match Utf8.Helper.width (String.unsafe_get s i) with | exception MalFormed -> loop (f acc i (malformed s i 1)) f s (i + 1) last | need -> let rem = last - i + 1 in if rem < need then f acc i (malformed s i rem) else loop (f acc i (r_utf_8 s i need)) f s (i + need) last) in let pos = 0 in let len = String.length s in let last = pos + len - 1 in loop acc f s pos last sedlex-3.8.1/src/syntax/utf8.mli000066400000000000000000000001561526113672700165310ustar00rootroot00000000000000val fold : f:('a -> int -> [> `Malformed of string | `Uchar of Uchar.t ] -> 'a) -> 'a -> string -> 'a sedlex-3.8.1/src/syntax/xml.ml000066400000000000000000000203161526113672700162720ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) open Sedlex_compiler.Cset (* Unicode classes from XML *) let base_char = let l = [ (0x0041, 0x005A); (0x0061, 0x007A); (0x00C0, 0x00D6); (0x00D8, 0x00F6); (0x00F8, 0x00FF); (0x0100, 0x0131); (0x0134, 0x013E); (0x0141, 0x0148); (0x014A, 0x017E); (0x0180, 0x01C3); (0x01CD, 0x01F0); (0x01F4, 0x01F5); (0x01FA, 0x0217); (0x0250, 0x02A8); (0x02BB, 0x02C1); (0x0386, 0x0386); (0x0388, 0x038A); (0x038C, 0x038C); (0x038E, 0x03A1); (0x03A3, 0x03CE); (0x03D0, 0x03D6); (0x03DA, 0x03DA); (0x03DC, 0x03DC); (0x03DE, 0x03DE); (0x03E0, 0x03E0); (0x03E2, 0x03F3); (0x0401, 0x040C); (0x040E, 0x044F); (0x0451, 0x045C); (0x045E, 0x0481); (0x0490, 0x04C4); (0x04C7, 0x04C8); (0x04CB, 0x04CC); (0x04D0, 0x04EB); (0x04EE, 0x04F5); (0x04F8, 0x04F9); (0x0531, 0x0556); (0x0559, 0x0559); (0x0561, 0x0586); (0x05D0, 0x05EA); (0x05F0, 0x05F2); (0x0621, 0x063A); (0x0641, 0x064A); (0x0671, 0x06B7); (0x06BA, 0x06BE); (0x06C0, 0x06CE); (0x06D0, 0x06D3); (0x06D5, 0x06D5); (0x06E5, 0x06E6); (0x0905, 0x0939); (0x093D, 0x093D); (0x0958, 0x0961); (0x0985, 0x098C); (0x098F, 0x0990); (0x0993, 0x09A8); (0x09AA, 0x09B0); (0x09B2, 0x09B2); (0x09B6, 0x09B9); (0x09DC, 0x09DD); (0x09DF, 0x09E1); (0x09F0, 0x09F1); (0x0A05, 0x0A0A); (0x0A0F, 0x0A10); (0x0A13, 0x0A28); (0x0A2A, 0x0A30); (0x0A32, 0x0A33); (0x0A35, 0x0A36); (0x0A38, 0x0A39); (0x0A59, 0x0A5C); (0x0A5E, 0x0A5E); (0x0A72, 0x0A74); (0x0A85, 0x0A8B); (0x0A8D, 0x0A8D); (0x0A8F, 0x0A91); (0x0A93, 0x0AA8); (0x0AAA, 0x0AB0); (0x0AB2, 0x0AB3); (0x0AB5, 0x0AB9); (0x0ABD, 0x0ABD); (0x0AE0, 0x0AE0); (0x0B05, 0x0B0C); (0x0B0F, 0x0B10); (0x0B13, 0x0B28); (0x0B2A, 0x0B30); (0x0B32, 0x0B33); (0x0B36, 0x0B39); (0x0B3D, 0x0B3D); (0x0B5C, 0x0B5D); (0x0B5F, 0x0B61); (0x0B85, 0x0B8A); (0x0B8E, 0x0B90); (0x0B92, 0x0B95); (0x0B99, 0x0B9A); (0x0B9C, 0x0B9C); (0x0B9E, 0x0B9F); (0x0BA3, 0x0BA4); (0x0BA8, 0x0BAA); (0x0BAE, 0x0BB5); (0x0BB7, 0x0BB9); (0x0C05, 0x0C0C); (0x0C0E, 0x0C10); (0x0C12, 0x0C28); (0x0C2A, 0x0C33); (0x0C35, 0x0C39); (0x0C60, 0x0C61); (0x0C85, 0x0C8C); (0x0C8E, 0x0C90); (0x0C92, 0x0CA8); (0x0CAA, 0x0CB3); (0x0CB5, 0x0CB9); (0x0CDE, 0x0CDE); (0x0CE0, 0x0CE1); (0x0D05, 0x0D0C); (0x0D0E, 0x0D10); (0x0D12, 0x0D28); (0x0D2A, 0x0D39); (0x0D60, 0x0D61); (0x0E01, 0x0E2E); (0x0E30, 0x0E30); (0x0E32, 0x0E33); (0x0E40, 0x0E45); (0x0E81, 0x0E82); (0x0E84, 0x0E84); (0x0E87, 0x0E88); (0x0E8A, 0x0E8A); (0x0E8D, 0x0E8D); (0x0E94, 0x0E97); (0x0E99, 0x0E9F); (0x0EA1, 0x0EA3); (0x0EA5, 0x0EA5); (0x0EA7, 0x0EA7); (0x0EAA, 0x0EAB); (0x0EAD, 0x0EAE); (0x0EB0, 0x0EB0); (0x0EB2, 0x0EB3); (0x0EBD, 0x0EBD); (0x0EC0, 0x0EC4); (0x0F40, 0x0F47); (0x0F49, 0x0F69); (0x10A0, 0x10C5); (0x10D0, 0x10F6); (0x1100, 0x1100); (0x1102, 0x1103); (0x1105, 0x1107); (0x1109, 0x1109); (0x110B, 0x110C); (0x110E, 0x1112); (0x113C, 0x113C); (0x113E, 0x113E); (0x1140, 0x1140); (0x114C, 0x114C); (0x114E, 0x114E); (0x1150, 0x1150); (0x1154, 0x1155); (0x1159, 0x1159); (0x115F, 0x1161); (0x1163, 0x1163); (0x1165, 0x1165); (0x1167, 0x1167); (0x1169, 0x1169); (0x116D, 0x116E); (0x1172, 0x1173); (0x1175, 0x1175); (0x119E, 0x119E); (0x11A8, 0x11A8); (0x11AB, 0x11AB); (0x11AE, 0x11AF); (0x11B7, 0x11B8); (0x11BA, 0x11BA); (0x11BC, 0x11C2); (0x11EB, 0x11EB); (0x11F0, 0x11F0); (0x11F9, 0x11F9); (0x1E00, 0x1E9B); (0x1EA0, 0x1EF9); (0x1F00, 0x1F15); (0x1F18, 0x1F1D); (0x1F20, 0x1F45); (0x1F48, 0x1F4D); (0x1F50, 0x1F57); (0x1F59, 0x1F59); (0x1F5B, 0x1F5B); (0x1F5D, 0x1F5D); (0x1F5F, 0x1F7D); (0x1F80, 0x1FB4); (0x1FB6, 0x1FBC); (0x1FBE, 0x1FBE); (0x1FC2, 0x1FC4); (0x1FC6, 0x1FCC); (0x1FD0, 0x1FD3); (0x1FD6, 0x1FDB); (0x1FE0, 0x1FEC); (0x1FF2, 0x1FF4); (0x1FF6, 0x1FFC); (0x2126, 0x2126); (0x212A, 0x212B); (0x212E, 0x212E); (0x2180, 0x2182); (0x3041, 0x3094); (0x30A1, 0x30FA); (0x3105, 0x312C); (0xAC00, 0xD7A3); ] in of_list l let ideographic = let l = [(0x3007, 0x3007); (0x3021, 0x3029); (0x4E00, 0x9FA5)] in of_list l let combining_char = let l = [ (0x0300, 0x0345); (0x0360, 0x0361); (0x0483, 0x0486); (0x0591, 0x05A1); (0x05A3, 0x05B9); (0x05BB, 0x05BD); (0x05BF, 0x05BF); (0x05C1, 0x05C2); (0x05C4, 0x05C4); (0x064B, 0x0652); (0x0670, 0x0670); (0x06D6, 0x06DC); (0x06DD, 0x06DF); (0x06E0, 0x06E4); (0x06E7, 0x06E8); (0x06EA, 0x06ED); (0x0901, 0x0903); (0x093C, 0x093C); (0x093E, 0x094C); (0x094D, 0x094D); (0x0951, 0x0954); (0x0962, 0x0963); (0x0981, 0x0983); (0x09BC, 0x09BC); (0x09BE, 0x09BE); (0x09BF, 0x09BF); (0x09C0, 0x09C4); (0x09C7, 0x09C8); (0x09CB, 0x09CD); (0x09D7, 0x09D7); (0x09E2, 0x09E3); (0x0A02, 0x0A02); (0x0A3C, 0x0A3C); (0x0A3E, 0x0A3E); (0x0A3F, 0x0A3F); (0x0A40, 0x0A42); (0x0A47, 0x0A48); (0x0A4B, 0x0A4D); (0x0A70, 0x0A71); (0x0A81, 0x0A83); (0x0ABC, 0x0ABC); (0x0ABE, 0x0AC5); (0x0AC7, 0x0AC9); (0x0ACB, 0x0ACD); (0x0B01, 0x0B03); (0x0B3C, 0x0B3C); (0x0B3E, 0x0B43); (0x0B47, 0x0B48); (0x0B4B, 0x0B4D); (0x0B56, 0x0B57); (0x0B82, 0x0B83); (0x0BBE, 0x0BC2); (0x0BC6, 0x0BC8); (0x0BCA, 0x0BCD); (0x0BD7, 0x0BD7); (0x0C01, 0x0C03); (0x0C3E, 0x0C44); (0x0C46, 0x0C48); (0x0C4A, 0x0C4D); (0x0C55, 0x0C56); (0x0C82, 0x0C83); (0x0CBE, 0x0CC4); (0x0CC6, 0x0CC8); (0x0CCA, 0x0CCD); (0x0CD5, 0x0CD6); (0x0D02, 0x0D03); (0x0D3E, 0x0D43); (0x0D46, 0x0D48); (0x0D4A, 0x0D4D); (0x0D57, 0x0D57); (0x0E31, 0x0E31); (0x0E34, 0x0E3A); (0x0E47, 0x0E4E); (0x0EB1, 0x0EB1); (0x0EB4, 0x0EB9); (0x0EBB, 0x0EBC); (0x0EC8, 0x0ECD); (0x0F18, 0x0F19); (0x0F35, 0x0F35); (0x0F37, 0x0F37); (0x0F39, 0x0F39); (0x0F3E, 0x0F3E); (0x0F3F, 0x0F3F); (0x0F71, 0x0F84); (0x0F86, 0x0F8B); (0x0F90, 0x0F95); (0x0F97, 0x0F97); (0x0F99, 0x0FAD); (0x0FB1, 0x0FB7); (0x0FB9, 0x0FB9); (0x20D0, 0x20DC); (0x20E1, 0x20E1); (0x302A, 0x302F); (0x3099, 0x3099); (0x309A, 0x309A); ] in of_list l let digit = let l = [ (0x0030, 0x0039); (0x0660, 0x0669); (0x06F0, 0x06F9); (0x0966, 0x096F); (0x09E6, 0x09EF); (0x0A66, 0x0A6F); (0x0AE6, 0x0AEF); (0x0B66, 0x0B6F); (0x0BE7, 0x0BEF); (0x0C66, 0x0C6F); (0x0CE6, 0x0CEF); (0x0D66, 0x0D6F); (0x0E50, 0x0E59); (0x0ED0, 0x0ED9); (0x0F20, 0x0F29); ] in of_list l let extender = let l = [ (0x00B7, 0x00B7); (0x02D0, 0x02D1); (0x0387, 0x0387); (0x0640, 0x0640); (0x0E46, 0x0E46); (0x0EC6, 0x0EC6); (0x3005, 0x3005); (0x3031, 0x3035); (0x309D, 0x309E); (0x30FC, 0x30FE); ] in of_list l let blank = let l = [(0x0009, 0x000A); (0x000D, 0x000D); (0x0020, 0x0020)] in of_list l let letter = union base_char ideographic sedlex-3.8.1/src/syntax/xml.mli000066400000000000000000000005201526113672700164360ustar00rootroot00000000000000(* The package sedlex is released under the terms of an MIT-like license. *) (* See the attached LICENSE file. *) (** Unicode classes from XML *) open Sedlex_compiler.Cset val letter : t val digit : t val extender : t val base_char : t val ideographic : t val combining_char : t val blank : t sedlex-3.8.1/test/000077500000000000000000000000001526113672700140005ustar00rootroot00000000000000sedlex-3.8.1/test/UTF-8-test.txt000066400000000000000000000543751526113672700163370ustar00rootroot00000000000000UTF-8 decoder capability and stress test ---------------------------------------- Markus Kuhn - 2015-08-28 - CC BY 4.0 This test file can help you examine, how your UTF-8 decoder handles various types of correct, malformed, or otherwise interesting UTF-8 sequences. This file is not meant to be a conformance test. It does not prescribe any particular outcome. Therefore, there is no way to "pass" or "fail" this test file, even though the text does suggest a preferable decoder behaviour at some places. Its aim is, instead, to help you think about, and test, the behaviour of your UTF-8 decoder on a systematic collection of unusual inputs. Experience so far suggests that most first-time authors of UTF-8 decoders find at least one serious problem in their decoder using this file. The test lines below cover boundary conditions, malformed UTF-8 sequences, as well as correctly encoded UTF-8 sequences of Unicode code points that should never occur in a correct UTF-8 file. According to ISO 10646-1:2000, sections D.7 and 2.3c, a device receiving UTF-8 shall interpret a "malformed sequence in the same way that it interprets a character that is outside the adopted subset" and "characters that are not within the adopted subset shall be indicated to the user" by a receiving device. One commonly used approach in UTF-8 decoders is to replace any malformed UTF-8 sequence by a replacement character (U+FFFD), which looks a bit like an inverted question mark, or a similar symbol. It might be a good idea to visually distinguish a malformed UTF-8 sequence from a correctly encoded Unicode character that is just not available in the current font but otherwise fully legal, even though ISO 10646-1 doesn't mandate this. In any case, just ignoring malformed sequences or unavailable characters does not conform to ISO 10646, will make debugging more difficult, and can lead to user confusion. Please check, whether a malformed UTF-8 sequence is (1) represented at all, (2) represented by exactly one single replacement character (or equivalent signal), and (3) the following quotation mark after an illegal UTF-8 sequence is correctly displayed, i.e. proper resynchronization takes place immediately after any malformed sequence. This file says "THE END" in the last line, so if you don't see that, your decoder crashed somehow before, which should always be cause for concern. All lines in this file are exactly 79 characters long (plus the line feed). In addition, all lines end with "|", except for the two test lines 2.1.1 and 2.2.1, which contain non-printable ASCII controls U+0000 and U+007F. If you display this file with a fixed-width font, these "|" characters should all line up in column 79 (right margin). This allows you to test quickly, whether your UTF-8 decoder finds the correct number of characters in every line, that is whether each malformed sequences is replaced by a single replacement character. Note that, as an alternative to the notion of malformed sequence used here, it is also a perfectly acceptable (and in some situations even preferable) solution to represent each individual byte of a malformed sequence with a replacement character. If you follow this strategy in your decoder, then please ignore the "|" column. Here come the tests: | | 1 Some correct UTF-8 text | | You should see the Greek word 'kosme': "κόσμε" | | 2 Boundary condition test cases | | 2.1 First possible sequence of a certain length | | 2.1.1 1 byte (U-00000000): "" 2.1.2 2 bytes (U-00000080): "€" | 2.1.3 3 bytes (U-00000800): "à €" | 2.1.4 4 bytes (U-00010000): "ð�€€" | 2.1.5 5 bytes (U-00200000): "øˆ€€€" | 2.1.6 6 bytes (U-04000000): "ü„€€€€" | | 2.2 Last possible sequence of a certain length | | 2.2.1 1 byte (U-0000007F): "" 2.2.2 2 bytes (U-000007FF): "ß¿" | 2.2.3 3 bytes (U-0000FFFF): "ï¿¿" | 2.2.4 4 bytes (U-001FFFFF): "÷¿¿¿" | 2.2.5 5 bytes (U-03FFFFFF): "û¿¿¿¿" | 2.2.6 6 bytes (U-7FFFFFFF): "ý¿¿¿¿¿" | | 2.3 Other boundary conditions | | 2.3.1 U-0000D7FF = ed 9f bf = "퟿" | 2.3.2 U-0000E000 = ee 80 80 = "" | 2.3.3 U-0000FFFD = ef bf bd = "�" | 2.3.4 U-0010FFFF = f4 8f bf bf = "ô�¿¿" | 2.3.5 U-00110000 = f4 90 80 80 = "ô�€€" | | 3 Malformed sequences | | 3.1 Unexpected continuation bytes | | Each unexpected continuation byte should be separately signalled as a | malformed sequence of its own. | | 3.1.1 First continuation byte 0x80: "€" | 3.1.2 Last continuation byte 0xbf: "¿" | | 3.1.3 2 continuation bytes: "€¿" | 3.1.4 3 continuation bytes: "€¿€" | 3.1.5 4 continuation bytes: "€¿€¿" | 3.1.6 5 continuation bytes: "€¿€¿€" | 3.1.7 6 continuation bytes: "€¿€¿€¿" | 3.1.8 7 continuation bytes: "€¿€¿€¿€" | | 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf): | | "€�‚ƒ„…†‡ˆ‰Š‹Œ�Ž� | �‘’“”•–—˜™š›œ�žŸ |  ¡¢£¤¥¦§¨©ª«¬­®¯ | °±²³´µ¶·¸¹º»¼½¾¿" | | 3.2 Lonely start characters | | 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf), | each followed by a space character: | | "À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï | Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß " | | 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef), | each followed by a space character: | | "à á â ã ä å æ ç è é ê ë ì í î ï " | | 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7), | each followed by a space character: | | "ð ñ ò ó ô õ ö ÷ " | | 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb), | each followed by a space character: | | "ø ù ú û " | | 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd), | each followed by a space character: | | "ü ý " | | 3.3 Sequences with last continuation byte missing | | All bytes of an incomplete sequence should be signalled as a single | malformed sequence, i.e., you should see only a single replacement | character in each of the next 10 tests. (Characters as in section 2) | | 3.3.1 2-byte sequence with last byte missing (U+0000): "À" | 3.3.2 3-byte sequence with last byte missing (U+0000): "à€" | 3.3.3 4-byte sequence with last byte missing (U+0000): "ð€€" | 3.3.4 5-byte sequence with last byte missing (U+0000): "ø€€€" | 3.3.5 6-byte sequence with last byte missing (U+0000): "ü€€€€" | 3.3.6 2-byte sequence with last byte missing (U-000007FF): "ß" | 3.3.7 3-byte sequence with last byte missing (U-0000FFFF): "ï¿" | 3.3.8 4-byte sequence with last byte missing (U-001FFFFF): "÷¿¿" | 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): "û¿¿¿" | 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "ý¿¿¿¿" | | 3.4 Concatenation of incomplete sequences | | All the 10 sequences of 3.3 concatenated, you should see 10 malformed | sequences being signalled: | | "Àà€ð€€ø€€€ü€€€€ßï¿÷¿¿û¿¿¿ý¿¿¿¿" | | 3.5 Impossible bytes | | The following two bytes cannot appear in a correct UTF-8 string | | 3.5.1 fe = "þ" | 3.5.2 ff = "ÿ" | 3.5.3 fe fe ff ff = "þþÿÿ" | | 4 Overlong sequences | | The following sequences are not malformed according to the letter of | the Unicode 2.0 standard. However, they are longer then necessary and | a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 | decoder" should reject them just like malformed sequences for two | reasons: (1) It helps to debug applications if overlong sequences are | not treated as valid representations of characters, because this helps | to spot problems more quickly. (2) Overlong sequences provide | alternative representations of characters, that could maliciously be | used to bypass filters that check only for ASCII characters. For | instance, a 2-byte encoded line feed (LF) would not be caught by a | line counter that counts only 0x0a bytes, but it would still be | processed as a line feed by an unsafe UTF-8 decoder later in the | pipeline. From a security point of view, ASCII compatibility of UTF-8 | sequences means also, that ASCII characters are *only* allowed to be | represented by ASCII bytes in the range 0x00-0x7f. To ensure this | aspect of ASCII compatibility, use only "safe UTF-8 decoders" that | reject overlong UTF-8 sequences for which a shorter encoding exists. | | 4.1 Examples of an overlong ASCII character | | With a safe UTF-8 decoder, all of the following five overlong | representations of the ASCII character slash ("/") should be rejected | like a malformed UTF-8 sequence, for instance by substituting it with | a replacement character. If you see a slash below, you do not have a | safe UTF-8 decoder! | | 4.1.1 U+002F = c0 af = "À¯" | 4.1.2 U+002F = e0 80 af = "à€¯" | 4.1.3 U+002F = f0 80 80 af = "ð€€¯" | 4.1.4 U+002F = f8 80 80 80 af = "ø€€€¯" | 4.1.5 U+002F = fc 80 80 80 80 af = "ü€€€€¯" | | 4.2 Maximum overlong sequences | | Below you see the highest Unicode value that is still resulting in an | overlong sequence if represented with the given number of bytes. This | is a boundary test for safe UTF-8 decoders. All five characters should | be rejected like malformed UTF-8 sequences. | | 4.2.1 U-0000007F = c1 bf = "Á¿" | 4.2.2 U-000007FF = e0 9f bf = "àŸ¿" | 4.2.3 U-0000FFFF = f0 8f bf bf = "ð�¿¿" | 4.2.4 U-001FFFFF = f8 87 bf bf bf = "ø‡¿¿¿" | 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf = "üƒ¿¿¿¿" | | 4.3 Overlong representation of the NUL character | | The following five sequences should also be rejected like malformed | UTF-8 sequences and should not be treated like the ASCII NUL | character. | | 4.3.1 U+0000 = c0 80 = "À€" | 4.3.2 U+0000 = e0 80 80 = "à€€" | 4.3.3 U+0000 = f0 80 80 80 = "ð€€€" | 4.3.4 U+0000 = f8 80 80 80 80 = "ø€€€€" | 4.3.5 U+0000 = fc 80 80 80 80 80 = "ü€€€€€" | | 5 Illegal code positions | | The following UTF-8 sequences should be rejected like malformed | sequences, because they never represent valid ISO 10646 characters and | a UTF-8 decoder that accepts them might introduce security problems | comparable to overlong UTF-8 sequences. | | 5.1 Single UTF-16 surrogates | | 5.1.1 U+D800 = ed a0 80 = "í €" | 5.1.2 U+DB7F = ed ad bf = "í­¿" | 5.1.3 U+DB80 = ed ae 80 = "í®€" | 5.1.4 U+DBFF = ed af bf = "í¯¿" | 5.1.5 U+DC00 = ed b0 80 = "í°€" | 5.1.6 U+DF80 = ed be 80 = "í¾€" | 5.1.7 U+DFFF = ed bf bf = "í¿¿" | | 5.2 Paired UTF-16 surrogates | | 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = "𐀀" | 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = "𐏿" | 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = "í­¿í°€" | 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = "í­¿í¿¿" | 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = "󰀀" | 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = "󰏿" | 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = "􏰀" | 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = "􏿿" | | 5.3 Noncharacter code positions | | The following "noncharacters" are "reserved for internal use" by | applications, and according to older versions of the Unicode Standard | "should never be interchanged". Unicode Corrigendum #9 dropped the | latter restriction. Nevertheless, their presence in incoming UTF-8 data | can remain a potential security risk, depending on what use is made of | these codes subsequently. Examples of such internal use: | | - Some file APIs with 16-bit characters may use the integer value -1 | = U+FFFF to signal an end-of-file (EOF) or error condition. | | - In some UTF-16 receivers, code point U+FFFE might trigger a | byte-swap operation (to convert between UTF-16LE and UTF-16BE). | | With such internal use of noncharacters, it may be desirable and safer | to block those code points in UTF-8 decoders, as they should never | occur legitimately in incoming UTF-8 data, and could trigger unsafe | behaviour in subsequent processing. | | Particularly problematic noncharacters in 16-bit applications: | | 5.3.1 U+FFFE = ef bf be = "￾" | 5.3.2 U+FFFF = ef bf bf = "ï¿¿" | | Other noncharacters: | | 5.3.3 U+FDD0 .. U+FDEF = "ï·�﷑﷒﷓﷔﷕﷖﷗﷘﷙﷚﷛﷜ï·�﷞﷟﷠﷡﷢﷣﷤﷥﷦﷧﷨﷩﷪﷫﷬﷭﷮﷯"| | 5.3.4 U+nFFFE U+nFFFF (for n = 1..10) | | "🿾🿿𯿾𯿿𿿾𿿿ñ�¿¾ñ�¿¿ñŸ¿¾ñŸ¿¿ñ¯¿¾ñ¯¿¿ñ¿¿¾ñ¿¿¿ò�¿¾ò�¿¿ | òŸ¿¾òŸ¿¿ò¯¿¾ò¯¿¿ò¿¿¾ò¿¿¿ó�¿¾ó�¿¿óŸ¿¾óŸ¿¿ó¯¿¾ó¯¿¿ó¿¿¾ó¿¿¿ô�¿¾ô�¿¿" | | THE END | sedlex-3.8.1/test/basic.ml000066400000000000000000002627731526113672700154340ustar00rootroot00000000000000let () = set_binary_mode_out stdout true let digit = [%sedlex.regexp? '0' .. '9'] let number = [%sedlex.regexp? Plus digit] let hex_digit = let digit = [%sedlex.regexp? '0' .. '9'] in let hex_letter = [%sedlex.regexp? 'a' .. 'f' | 'A' .. 'F'] in [%sedlex.regexp? digit | hex_letter] let print_pos buf = let f { Lexing.pos_lnum; pos_bol; pos_cnum; _ } = Printf.sprintf "line=%d:bol=%d:cnum=%d" pos_lnum pos_bol pos_cnum in let f ~prefix (startp, endp) = Printf.printf "%s pos: [%s;%s]\n" prefix (f startp) (f endp) in f ~prefix:"code point" (Sedlexing.lexing_positions buf); f ~prefix:"bytes" (Sedlexing.lexing_bytes_positions buf) let rec token buf = match%sedlex buf with | number -> print_pos buf; Printf.printf "Number %s\n" (Sedlexing.Utf8.lexeme buf); token buf | id_start, Star id_continue -> print_pos buf; Printf.printf "Ident %s\n" (Sedlexing.Utf8.lexeme buf); token buf | Plus xml_blank -> token buf | Plus (Chars "+*-/") -> print_pos buf; Printf.printf "Op %s\n" (Sedlexing.Utf8.lexeme buf); token buf | eof -> print_pos buf; print_endline "EOF" | any -> print_pos buf; Printf.printf "Any %s\n" (Sedlexing.Utf8.lexeme buf); token buf | _ -> assert false let utf16_of_utf8 ?(endian = Sedlexing.Utf16.Big_endian) s = let b = Buffer.create (String.length s * 4) in let rec loop pos = if pos >= String.length s then () else ( let c = String.get_utf_8_uchar s pos in let u = Uchar.utf_decode_uchar c in (match endian with | Big_endian -> Buffer.add_utf_16be_uchar b u | Little_endian -> Buffer.add_utf_16le_uchar b u); loop (pos + Uchar.utf_decode_length c)) in loop 0; Buffer.contents b let remove_last s n = String.sub s 0 (String.length s - n) let gen_from_string s = let i = ref 0 in fun () -> if !i >= String.length s then None else ( let c = String.get s !i in incr i; Some c) let channel_from_string s = let name, oc = Filename.open_temp_file "" "" in output_string oc s; close_out oc; open_in_bin name let test_latin s f = print_endline "== from_string =="; (try f (Sedlexing.Latin1.from_string s) with Sedlexing.MalFormed -> print_endline "MalFormed"); print_endline "== from_gen =="; (try f (Sedlexing.Latin1.from_gen (gen_from_string s)) with Sedlexing.MalFormed -> print_endline "MalFormed"); print_endline "== from_channel =="; try f (Sedlexing.Latin1.from_channel (channel_from_string s)) with Sedlexing.MalFormed -> print_endline "MalFormed" let test_utf8 s f = print_endline "== from_string =="; (try f (Sedlexing.Utf8.from_string s) with Sedlexing.MalFormed -> print_endline "MalFormed"); print_endline "== from_gen =="; (try f (Sedlexing.Utf8.from_gen (gen_from_string s)) with Sedlexing.MalFormed -> print_endline "MalFormed"); print_endline "== from_channel =="; try f (Sedlexing.Utf8.from_channel (channel_from_string s)) with Sedlexing.MalFormed -> print_endline "MalFormed" let test_utf16 s bo f = print_endline "== from_string =="; (try f (Sedlexing.Utf16.from_string s bo) with Sedlexing.MalFormed -> print_endline "MalFormed"); print_endline "== from_gen =="; (try f (Sedlexing.Utf16.from_gen (gen_from_string s) bo) with Sedlexing.MalFormed -> print_endline "MalFormed"); print_endline "== from_channel =="; try f (Sedlexing.Utf16.from_channel (channel_from_string s) bo) with Sedlexing.MalFormed -> print_endline "MalFormed" let%expect_test "latin1" = let s = "asas 123 + 2asd" in test_latin s (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] EOF |}]; let s = "asas 123 + 2\129" in test_latin s (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] Any Â� code point pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=13] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] Any Â� code point pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=13] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] Any Â� code point pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=13] EOF |}] let%expect_test "utf8" = let s = {|as🎉as 123 + 2asd|} in test_utf8 s (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] Ident as code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=3] bytes pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=6] Any 🎉 code point pos: [line=1:bol=0:cnum=3;line=1:bol=0:cnum=5] bytes pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=8] Ident as code point pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=9] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=12] Number 123 code point pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=11] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=14] Op + code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=16] Number 2 code point pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=16] bytes pos: [line=1:bol=0:cnum=16;line=1:bol=0:cnum=19] Ident asd code point pos: [line=1:bol=0:cnum=16;line=1:bol=0:cnum=16] bytes pos: [line=1:bol=0:cnum=19;line=1:bol=0:cnum=19] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] Ident as code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=3] bytes pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=6] Any 🎉 code point pos: [line=1:bol=0:cnum=3;line=1:bol=0:cnum=5] bytes pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=8] Ident as code point pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=9] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=12] Number 123 code point pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=11] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=14] Op + code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=16] Number 2 code point pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=16] bytes pos: [line=1:bol=0:cnum=16;line=1:bol=0:cnum=19] Ident asd code point pos: [line=1:bol=0:cnum=16;line=1:bol=0:cnum=16] bytes pos: [line=1:bol=0:cnum=19;line=1:bol=0:cnum=19] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] Ident as code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=3] bytes pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=6] Any 🎉 code point pos: [line=1:bol=0:cnum=3;line=1:bol=0:cnum=5] bytes pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=8] Ident as code point pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=9] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=12] Number 123 code point pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=11] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=14] Op + code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=13] bytes pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=16] Number 2 code point pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=16] bytes pos: [line=1:bol=0:cnum=16;line=1:bol=0:cnum=19] Ident asd code point pos: [line=1:bol=0:cnum=16;line=1:bol=0:cnum=16] bytes pos: [line=1:bol=0:cnum=19;line=1:bol=0:cnum=19] EOF |}]; let s = {|as🎉as 123 + 2|} ^ "\129" in test_utf8 s (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] Ident as code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=3] bytes pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=6] Any 🎉 code point pos: [line=1:bol=0:cnum=3;line=1:bol=0:cnum=5] bytes pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=8] Ident as code point pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=9] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=12] Number 123 code point pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=11] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=14] Op + MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] Ident as code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=3] bytes pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=6] Any 🎉 code point pos: [line=1:bol=0:cnum=3;line=1:bol=0:cnum=5] bytes pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=8] Ident as code point pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=9] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=12] Number 123 code point pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=11] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=14] Op + MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] Ident as code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=3] bytes pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=6] Any 🎉 code point pos: [line=1:bol=0:cnum=3;line=1:bol=0:cnum=5] bytes pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=8] Ident as code point pos: [line=1:bol=0:cnum=6;line=1:bol=0:cnum=9] bytes pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=12] Number 123 code point pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=11] bytes pos: [line=1:bol=0:cnum=13;line=1:bol=0:cnum=14] Op + MalFormed |}] let%expect_test "utf16" = let bo = None in let s = utf16_of_utf8 "asas 123 + 2asd" in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF |}]; let s = utf16_of_utf8 "asas 123 + 2" ^ "a" in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed |}]; let s1 = "12asd12\u{1F6F3}" in let s = utf16_of_utf8 s1 in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF |}]; test_utf16 (remove_last s 1) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 2) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 3) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 4) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF |}] let%expect_test "utf16-be" = let endian = Sedlexing.Utf16.Big_endian in let utf16_of_utf8 = utf16_of_utf8 ~endian in let bo = Some endian in let s = utf16_of_utf8 "asas 123 + 2asd" in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF |}]; let s = utf16_of_utf8 "asas 123 + 2" ^ "a" in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed |}]; let s1 = "12asd12\u{1F6F3}" in let s = utf16_of_utf8 s1 in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF |}]; test_utf16 (remove_last s 1) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 2) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 3) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 4) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF |}] let%expect_test "utf16-le" = let endian = Sedlexing.Utf16.Little_endian in let utf16_of_utf8 = utf16_of_utf8 ~endian in let bo = Some endian in let s = utf16_of_utf8 "asas 123 + 2asd" in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + code point pos: [line=1:bol=0:cnum=11;line=1:bol=0:cnum=12] bytes pos: [line=1:bol=0:cnum=22;line=1:bol=0:cnum=24] Number 2 code point pos: [line=1:bol=0:cnum=12;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=24;line=1:bol=0:cnum=30] Ident asd code point pos: [line=1:bol=0:cnum=15;line=1:bol=0:cnum=15] bytes pos: [line=1:bol=0:cnum=30;line=1:bol=0:cnum=30] EOF |}]; let s = utf16_of_utf8 "asas 123 + 2" ^ "a" in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=8] Ident asas code point pos: [line=1:bol=0:cnum=5;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=10;line=1:bol=0:cnum=16] Number 123 code point pos: [line=1:bol=0:cnum=9;line=1:bol=0:cnum=10] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=20] Op + MalFormed |}]; let s1 = "12asd12\u{1F6F3}" in let s = utf16_of_utf8 s1 in test_utf16 s bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=18] Any 🛳 code point pos: [line=1:bol=0:cnum=8;line=1:bol=0:cnum=8] bytes pos: [line=1:bol=0:cnum=18;line=1:bol=0:cnum=18] EOF |}]; test_utf16 (remove_last s 1) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 2) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 3) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 MalFormed |}]; test_utf16 (remove_last s 4) bo (fun lb -> token lb); [%expect {| == from_string == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF == from_gen == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF == from_channel == code point pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=2] bytes pos: [line=1:bol=0:cnum=0;line=1:bol=0:cnum=4] Number 12 code point pos: [line=1:bol=0:cnum=2;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=4;line=1:bol=0:cnum=14] Ident asd12 code point pos: [line=1:bol=0:cnum=7;line=1:bol=0:cnum=7] bytes pos: [line=1:bol=0:cnum=14;line=1:bol=0:cnum=14] EOF |}] let%expect_test "utf8 surrogate rejection" = (* UTF-16 surrogates (U+D800..U+DFFF) must be rejected as invalid UTF-8 *) let test s = try let lb = Sedlexing.Utf8.from_string s in ignore (Sedlexing.__private__next_int lb); Printf.printf "accepted (BUG)\n" with Sedlexing.MalFormed -> Printf.printf "rejected\n" in (* U+D800: first high surrogate *) test "\xED\xA0\x80"; (* U+DF01: low surrogate *) test "\xED\xBC\x81"; (* U+DFFF: last low surrogate *) test "\xED\xBF\xBF"; [%expect {| rejected rejected rejected |}] let%expect_test "nested_let_regexp" = let int_lit = let digit = [%sedlex.regexp? '0' .. '9'] in [%sedlex.regexp? Plus digit] in let buf = Sedlexing.Utf8.from_string "123abc" in let rec loop () = match%sedlex buf with | int_lit -> Printf.printf "Int: %s\n" (Sedlexing.Utf8.lexeme buf); loop () | Plus 'a' .. 'z' -> Printf.printf "Word: %s\n" (Sedlexing.Utf8.lexeme buf); loop () | eof -> Printf.printf "EOF\n" | _ -> assert false in loop (); [%expect {| Int: 123 Word: abc EOF |}] let%expect_test "nested_let_regexp_toplevel" = let buf = Sedlexing.Utf8.from_string "0xDEAD rest" in let rec loop () = match%sedlex buf with | "0x", Plus hex_digit -> Printf.printf "Hex: %s\n" (Sedlexing.Utf8.lexeme buf); loop () | Plus 'a' .. 'z' -> Printf.printf "Word: %s\n" (Sedlexing.Utf8.lexeme buf); loop () | ' ' -> loop () | eof -> Printf.printf "EOF\n" | _ -> assert false in loop (); [%expect {| Hex: 0xDEAD Word: rest EOF |}] let letter = [%sedlex.regexp? 'a' .. 'z' | 'A' .. 'Z'] let%expect_test "as_bindings" = (* Test 1: simple binding in middle of sequence *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | 'a', ('b' as x), 'c' -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=b |}]; (* Test 2: multiple bindings *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | ('a' as x), ('b' as y), 'c' -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=a y=b |}]; (* Test 3: binding with named regexp *) let buf = Sedlexing.Utf8.from_string "123z" in (match%sedlex buf with | number, (letter as x) -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=z |}]; (* Test 4: whole-match binding *) let buf = Sedlexing.Utf8.from_string "hello" in (match%sedlex buf with | Plus 'a' .. 'z' as x -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=hello |}]; (* Test 5: multi-char UTF-8 *) let buf = Sedlexing.Utf8.from_string "a\xC3\xA9b" in (match%sedlex buf with | 'a', (any as x), 'b' -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=é |}]; (* Test 6: variable-length named segment *) let buf = Sedlexing.Utf8.from_string {|"hello"|} in (match%sedlex buf with | '"', (Star (Compl '"') as content), '"' -> Printf.printf "content=%s\n" (Sedlexing.Utf8.of_submatch content) | _ -> assert false); [%expect {| content=hello |}]; (* Test 7: as binding wrapping an alternation *) let buf = Sedlexing.Utf8.from_string "xb" in (match%sedlex buf with | 'x', (('a' | 'b') as x) -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=b |}]; (* Test 8: as binding in both branches of or-pattern *) let buf = Sedlexing.Utf8.from_string "123" in (match%sedlex buf with | (number as x) | (Plus letter as x) -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=123 |}]; let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | (number as x) | (Plus letter as x) -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=abc |}]; (* Test 9: as binding inside or, in a sequence *) let buf = Sedlexing.Utf8.from_string "<42>" in (match%sedlex buf with | '<', ((number as x) | (Plus letter as x)), '>' -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=42 |}]; let buf = Sedlexing.Utf8.from_string "" in (match%sedlex buf with | '<', ((number as x) | (Plus letter as x)), '>' -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=hello |}]; (* Test 10: or-pattern with shared prefix requiring discriminator tags *) let buf = Sedlexing.Utf8.from_string "abcdef" in (match%sedlex buf with | ("abc" as x), "def" | "a", ("bcd" as x), "ey" -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=abc |}]; let buf = Sedlexing.Utf8.from_string "abcdey" in (match%sedlex buf with | ("abc" as x), "def" | "a", ("bcd" as x), "ey" -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=bcd |}]; (* Test 10b: 3-way or-pattern reuses a single disc cell; the first and third branches bind x at the same offsets and share a disc value *) let buf = Sedlexing.Utf8.from_string "abcd" in (match%sedlex buf with | ("ab" as x), "cd" | ("a" as x), "bce" | ("abc" as x), "df" -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=ab |}]; let buf = Sedlexing.Utf8.from_string "abce" in (match%sedlex buf with | ("ab" as x), "cd" | ("a" as x), "bce" | ("abc" as x), "df" -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=a |}]; let buf = Sedlexing.Utf8.from_string "abcdf" in (match%sedlex buf with | ("ab" as x), "cd" | ("a" as x), "bce" | ("abc" as x), "df" -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=abc |}]; (* Test 10c: or-pattern with inner or + extra binding without disc *) let buf = Sedlexing.Utf8.from_string "aef" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | ("cd" as x), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=a y=ef |}]; let buf = Sedlexing.Utf8.from_string "bef" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | ("cd" as x), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=b y=ef |}]; let buf = Sedlexing.Utf8.from_string "cdgh" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | ("cd" as x), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=cd y=gh |}]; (* Test 10d: nested or-patterns on both sides *) let buf = Sedlexing.Utf8.from_string "aef" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | (("c" as x) | ("d" as x)), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=a y=ef |}]; let buf = Sedlexing.Utf8.from_string "bef" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | (("c" as x) | ("d" as x)), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=b y=ef |}]; let buf = Sedlexing.Utf8.from_string "cgh" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | (("c" as x) | ("d" as x)), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=c y=gh |}]; let buf = Sedlexing.Utf8.from_string "dgh" in (match%sedlex buf with | (("a" as x) | ("b" as x)), ("ef" as y) | (("c" as x) | ("d" as x)), ("gh" as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=d y=gh |}]; (* Test 11: delayed tag with backtracking (Opt at end) *) let buf = Sedlexing.Utf8.from_string "aabba" in (match%sedlex buf with | (Plus 'a' as x), ((Plus 'b', Opt 'a') as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=aa y=bba |}]; let buf = Sedlexing.Utf8.from_string "aabb" in (match%sedlex buf with | (Plus 'a' as x), ((Plus 'b', Opt 'a') as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=aa y=bb |}]; let buf = Sedlexing.Utf8.from_string "aba" in (match%sedlex buf with | (Plus 'a' as x), ((Plus 'b', Opt 'a') as y) -> Printf.printf "x=%s y=%s\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y) | _ -> assert false); [%expect {| x=a y=ba |}] let num_mem buf = Sedlexing.__private__num_mem_cells buf let%expect_test "as_bindings_num_mem_cells" = (* No bindings: 0 cells *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | "abc" -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Single binding in tuple: 0 cells (prefix + suffix known) *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | 'a', ('b' as _x), 'c' -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Two bindings in tuple: 0 cells (all offsets known) *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | ('a' as _x), ('b' as _y), 'c' -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Whole-match binding: 0 cells (Start_plus 0, End_minus 0) *) let buf = Sedlexing.Utf8.from_string "hello" in (match%sedlex buf with | Plus 'a' .. 'z' as _x -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* as wrapping alternation in tuple: 0 cells (prefix=1, suffix=0 known) *) let buf = Sedlexing.Utf8.from_string "xb" in (match%sedlex buf with | 'x', (('a' | 'b') as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Or-pattern with identical offsets: 0 cells (disc elided) *) let buf = Sedlexing.Utf8.from_string "123" in (match%sedlex buf with | (number as _x) | (Plus letter as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Or-pattern with different offsets: positions are known, so only the discriminator needs a cell. Its write is delayed until the branch reaches its final node, where the final operations set the canonical cell directly: no working register *) let buf = Sedlexing.Utf8.from_string "abcdef" in (match%sedlex buf with | ("abc" as _x), "def" | "a", ("bcd" as _x), "ey" -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=1 |}] let%expect_test "as_bindings_multi_rule_mem_cells" = (* All rules in a match%sedlex share one pool of memory cells. The total is the sum of tags across ALL rules, not just the matched one. *) (* One rule with binding in tuple, one without: 0 cells (offsets known) *) let buf = Sedlexing.Utf8.from_string "ab" in (match%sedlex buf with | 'a', ('b' as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | "cd" -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Even when the no-binding rule matches, no cells needed (offsets known) *) let buf = Sedlexing.Utf8.from_string "cd" in (match%sedlex buf with | 'a', ('b' as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | "cd" -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Two rules, each with one binding: 0 cells (all offsets known) *) let buf = Sedlexing.Utf8.from_string "ab" in (match%sedlex buf with | 'a', ('b' as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | 'c', ('d' as _y) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Three rules with one binding each: 0 cells (all offsets known) *) let buf = Sedlexing.Utf8.from_string "ab" in (match%sedlex buf with | 'a', ('b' as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | 'c', ('d' as _y) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | 'e', ('f' as _z) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Whole-match + tuple binding: 0 cells (all offsets known) *) let buf = Sedlexing.Utf8.from_string "hello" in (match%sedlex buf with | Plus 'a' .. 'z' as _x -> Printf.printf "mem_cells=%d\n" (num_mem buf) | '0', (number as _y) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Or-pattern (disc elided) + tuple binding (offsets known): 0 cells *) let buf = Sedlexing.Utf8.from_string "123" in (match%sedlex buf with | (number as _x) | (Plus letter as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | '{', (Star (Compl '}') as _y), '}' -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Two or-pattern rules (both disc elided): 0 cells *) let buf = Sedlexing.Utf8.from_string "123" in (match%sedlex buf with | (number as _x) | (Plus letter as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | ('<' as _y) | ('>' as _y) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=0 |}]; (* Two rules that each need a tag: the pool holds both, whichever rule matches (a tag costs its canonical cell plus a working register) *) let lex buf = match%sedlex buf with | Plus 'a', (Plus 'b' as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | Plus 'c', (Plus 'd' as _y) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false in lex (Sedlexing.Utf8.from_string "ab"); [%expect {| mem_cells=4 |}]; lex (Sedlexing.Utf8.from_string "cd"); [%expect {| mem_cells=4 |}] let%expect_test "as_bindings_nested_sedlex" = (* Regression: a nested match%sedlex in a case RHS must not reset the outer match's tag counter, which would cause ensure_mem/set_mem to be dropped and as-bindings to read uninitialized memory cells. The outer bindings need a tag (variable-length prefix). *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | Plus 'a', ('b' as x), Plus 'c' -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | Star any -> ( (* Nested match%sedlex in a case RHS *) Sedlexing.rollback buf; match%sedlex buf with | Plus 'a' .. 'z' -> Printf.printf "word\n" | _ -> Printf.printf "other\n") | _ -> assert false); [%expect {| x=b |}]; (* Same but the nested match comes in a case BEFORE the as-binding rule *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | '0' .. '9' -> ( Sedlexing.rollback buf; match%sedlex buf with | '0' .. '9' -> Printf.printf "digit\n" | _ -> Printf.printf "other\n") | Plus 'a', (Plus 'b' .. 'z' as x) -> Printf.printf "x=%s\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| x=bc |}]; (* Verify the outer match still allocates memory cells *) let buf = Sedlexing.Utf8.from_string "abc" in (match%sedlex buf with | '0' .. '9' -> ( Sedlexing.rollback buf; match%sedlex buf with | '0' .. '9' -> Printf.printf "digit\n" | _ -> Printf.printf "other\n") | Plus 'a', (Plus 'b' .. 'z' as _x) -> Printf.printf "mem_cells=%d\n" (num_mem buf) | _ -> assert false); [%expect {| mem_cells=2 |}] (* ------------------------------------------------------------------------ *) (* Regression tests. They were first pinned to the behavior of the time and flipped by the fixes; the comment above each case says what used to go wrong. *) (* Regression (#199, repetition loop before a capture): the epsilon closure used to re-fire the capture's start tag on every iteration of a preceding Star, so the recorded start drifted to the last iteration. *) let%expect_test "loop_before_capture" = let sub = Sedlexing.Latin1.of_submatch in (* expected x="abb" *) let buf = Sedlexing.Latin1.from_string "aabb" in (match%sedlex buf with | Star 'a', (('a', Plus 'b') as x) -> Printf.printf "x=%S\n" (sub x) | _ -> print_endline "nomatch"); [%expect {| x="abb" |}]; (* expected x="a" *) let buf = Sedlexing.Latin1.from_string "aab" in (match%sedlex buf with | Star 'a', (Plus 'a' as x), 'b' -> Printf.printf "x=%S\n" (sub x) | _ -> print_endline "nomatch"); [%expect {| x="a" |}]; (* the next two were never affected and pin the edge of the bug *) let buf = Sedlexing.Latin1.from_string "aba" in (match%sedlex buf with | Star 'a', (('a', 'a' .. 'c' | Star 'a' .. 'c') as y) -> Printf.printf "y=%S\n" (sub y) | _ -> print_endline "nomatch"); [%expect {| y="ba" |}]; let buf = Sedlexing.Latin1.from_string "bb" in (match%sedlex buf with | Star 'a', ('b' as y), Star 'b' -> Printf.printf "y=%S\n" (sub y) | _ -> print_endline "nomatch"); [%expect {| y="b" |}] (* [eof] is zero-width at runtime — [next] reports it without advancing — so the static offset optimization for captures must count it as zero code points, and a cset mixing [eof] with a real character has no fixed width at all. *) let%expect_test "capture_before_eof" = let sub x = try Printf.sprintf "%S" (Sedlexing.Latin1.of_submatch x) with e -> "raises " ^ Printexc.to_string e in (* expected x="abc" *) let buf = Sedlexing.Latin1.from_string "abc" in (match%sedlex buf with | (Star any as x), eof -> Printf.printf "x=%s\n" (sub x) | _ -> print_endline "nomatch"); [%expect {| x="abc" |}]; (* expected x="a" *) let buf = Sedlexing.Latin1.from_string "a" in (match%sedlex buf with | ('a' as x), eof -> Printf.printf "x=%s\n" (sub x) | _ -> print_endline "nomatch"); [%expect {| x="a" |}]; (* expected x="" *) let buf = Sedlexing.Latin1.from_string "b" in (match%sedlex buf with | Star any, (Opt 'a' as x), eof -> Printf.printf "x=%s\n" (sub x) | _ -> print_endline "nomatch"); [%expect {| x="" |}]; (* expected x="x" on both inputs: the mixed cset is 0 or 1 wide *) let lex buf = match%sedlex buf with | ('x' as x), ('a' | eof) -> Printf.printf "x=%s\n" (sub x) | _ -> print_endline "nomatch" in lex (Sedlexing.Latin1.from_string "x"); [%expect {| x="x" |}]; lex (Sedlexing.Latin1.from_string "xa"); [%expect {| x="x" |}] (* End of input counts as one more symbol read: a rule matching [s, eof] beats an earlier rule matching [s]. 3.8 let declaration order decide instead, and a lexer with a nullable rule before [eof] looped forever. *) let%expect_test "eof_longest_match" = let inputs = ["cc"; "c"; ""; "a"] in let run lex = List.iter (fun s -> Printf.printf "%-4S -> %s\n" s (lex (Sedlexing.Utf8.from_string s))) inputs in let lex buf = match%sedlex buf with | Plus ('b' | 'c') -> "rule0" | Star 'a' .. 'c', eof -> "rule1" | _ -> "none" in run lex; [%expect {| "cc" -> rule1 "c" -> rule1 "" -> rule1 "a" -> rule1 |}]; (* mirror: the eof-terminated rule is declared first *) let lex_eof_first buf = match%sedlex buf with | Star 'a' .. 'c', eof -> "rule0" | Plus ('b' | 'c') -> "rule1" | _ -> "none" in run lex_eof_first; [%expect {| "cc" -> rule0 "c" -> rule0 "" -> rule0 "a" -> rule0 |}]; (* two rules reading eof: declaration order decides *) let lex_both buf = match%sedlex buf with | Plus ('b' | 'c'), eof -> "rule0" | Star 'a' .. 'c', eof -> "rule1" | _ -> "none" in run lex_both; [%expect {| "cc" -> rule0 "c" -> rule0 "" -> rule1 "a" -> rule1 |}]; (* a nullable rule before [eof], the usual shape of a lexer loop *) let rec tokens fuel acc buf = if fuel = 0 then "loops" else ( match%sedlex buf with | '@' -> tokens (fuel - 1) ("@" :: acc) buf | Star (Sub (any, '@')) -> tokens (fuel - 1) (Sedlexing.Utf8.lexeme buf :: acc) buf | eof -> String.concat " " (List.rev_map (Printf.sprintf "%S") acc) | _ -> assert false) in List.iter (fun s -> Printf.printf "%-8S -> %s\n" s (tokens 100 [] (Sedlexing.Utf8.from_string s))) ["ab@cd"; "@@"; ""]; [%expect {| "ab@cd" -> "ab" "@" "cd" "@@" -> "@" "@" "" -> |}]; (* the final operations of the rule accepting before eof are overridden *) let sub = Sedlexing.Latin1.of_submatch in let lex buf = match%sedlex buf with | Plus 'a' as x -> Printf.printf "rule0 x=%S\n" (sub x) | (Star 'a' as x), 'a', eof -> Printf.printf "rule1 x=%S\n" (sub x) | _ -> print_endline "nomatch" in lex (Sedlexing.Latin1.from_string "aaa"); [%expect {| rule1 x="aa" |}]; lex (Sedlexing.Latin1.from_string "aab"); [%expect {| rule0 x="aa" |}] (* eof within a rule: the leftmost-greedy parse only becomes accepting through the [eof] arm, after the state reached by the last real character has already marked a parse of the same rule with a shorter capture. The parse reading eof is the longer one: its transition returns directly over the mark. *) let%expect_test "eof_zero_width_tie_within_rule" = let sub = Sedlexing.Latin1.of_submatch in (* expected x="aaa" and x="a" *) let lex buf = match%sedlex buf with | (Star 'a' as x), ('a' | eof) -> Printf.printf "x=%S\n" (sub x) | _ -> print_endline "nomatch" in lex (Sedlexing.Latin1.from_string "aaa"); [%expect {| x="aaa" |}]; lex (Sedlexing.Latin1.from_string "a"); [%expect {| x="a" |}]; (* expected x="b" and x="cb" *) let lex buf = match%sedlex buf with | (Star 'b' .. 'd' as x), ('b' | eof) -> Printf.printf "x=%S\n" (sub x) | _ -> print_endline "nomatch" in lex (Sedlexing.Latin1.from_string "b"); [%expect {| x="b" |}]; lex (Sedlexing.Latin1.from_string "cb"); [%expect {| x="cb" |}]; (* expected x="a": the left branch of the or-pattern parses "ab" too *) let buf = Sedlexing.Latin1.from_string "ab" in (match%sedlex buf with | ('a' as x), 'b', eof | 'a', ('b' as x) -> Printf.printf "x=%S\n" (sub x) | _ -> print_endline "nomatch"); [%expect {| x="a" |}]; (* mirror: the right branch reads eof and beats the leftmost one *) let lex buf = match%sedlex buf with | 'a', ('b' as x) | ('a' as x), 'b', eof -> Printf.printf "x=%S\n" (sub x) | _ -> print_endline "nomatch" in lex (Sedlexing.Latin1.from_string "ab"); [%expect {| x="a" |}]; lex (Sedlexing.Latin1.from_string "abc"); [%expect {| x="b" |}] (* Regression (eof self-loop): end of input is read once. [eof] used to be reported again and again without advancing, so an accepting state with an [eof] transition back to itself spun forever at end of input, and [eof, eof] matched. *) let%expect_test "eof_self_loop_terminates" = let run lex inputs = List.iter (fun s -> let buf = Sedlexing.Latin1.from_string s in let r = lex buf in Printf.printf "%-4S -> %S %s\n" s (Sedlexing.Latin1.lexeme buf) r) inputs in run (fun buf -> match%sedlex buf with Plus eof -> "rule0" | _ -> "none") [""; "a"]; [%expect {| "" -> "" rule0 "a" -> "" none |}]; run (fun buf -> match%sedlex buf with Star ('a' | eof) -> "rule0" | _ -> "none") ["aa"; ""; "ab"]; [%expect {| "aa" -> "aa" rule0 "" -> "" rule0 "ab" -> "a" rule0 |}]; run (fun buf -> match%sedlex buf with Star 'a', Star eof -> "rule0" | _ -> "none") ["a"; ""]; [%expect {| "a" -> "a" rule0 "" -> "" rule0 |}]; (* the second eof has nothing left to read *) run (fun buf -> match%sedlex buf with eof, eof -> "rule0" | _ -> "none") [""]; [%expect {| "" -> "" none |}] (* Regression (#199, capture before a repetition loop): the mirror of [loop_before_capture]. The capture's end tag used to keep firing inside the repetition that follows it, so the submatch swallowed characters the rest of the rule then consumed, and the reported parse was one that cannot exist. *) let%expect_test "capture_before_loop" = let sub x = try Printf.sprintf "%S" (Sedlexing.Latin1.of_submatch x) with e -> "raises " ^ Printexc.to_string e in let run inputs lex = List.iter (fun s -> Printf.printf "%-7S -> %s\n" s (lex (Sedlexing.Latin1.from_string s))) inputs in (* expected z="aa" and z="a": the Plus needs at least one 'a' *) run ["aaa"; "aa"] (fun buf -> match%sedlex buf with | (Rep ('a' .. 'c', 0 .. 2) as z), Plus 'a' -> "z=" ^ sub z | _ -> "nomatch"); [%expect {| "aaa" -> z="aa" "aa" -> z="a" |}]; (* expected z="a": the tail needs an even number of characters *) run ["aaaaa"] (fun buf -> match%sedlex buf with | (Rep ('a', 0 .. 2) as z), Plus ('a', 'a') -> "z=" ^ sub z | _ -> "nomatch"); [%expect {| "aaaaa" -> z="a" |}]; (* expected x="bd": one character is left for the middle class *) run ["bdc"] (fun buf -> match%sedlex buf with | (Plus 'b' .. 'd' as x), 'a' .. 'd', Star 'c' .. 'd' -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "bdc" -> x="bd" |}]; (* expected x="b" y="b": y cannot be empty *) run ["bbac"] (fun buf -> match%sedlex buf with | (Rep ('b', 1 .. 3) as x), (Plus 'b' as y) -> "x=" ^ sub x ^ " y=" ^ sub y | _ -> "nomatch"); [%expect {| "bbac" -> x="b" y="b" |}]; (* expected x="c" *) run ["ccd"] (fun buf -> match%sedlex buf with | (Star (Star 'c') as x), Plus 'a' .. 'c' -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "ccd" -> x="c" |}]; (* expected x="a": the parse is ambiguous and the Star is greedy *) run ["aaa"] (fun buf -> match%sedlex buf with | Star 'a', (Plus 'a' as x) -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "aaa" -> x="a" |}]; (* expected y="d" w="": both submatches used to have garbage bounds, and extracting them raised *) run ["ddd"] (fun buf -> match%sedlex buf with | Star 'd', ('a' .. 'd' as y), 'a' .. 'd', ((Star 'a' | 'b') as w) -> "y=" ^ sub y ^ " w=" ^ sub w | _ -> "nomatch"); [%expect {| "ddd" -> y="d" w="" |}]; (* the fixed-width tail was never affected and pins the edge of the bug *) run ["aa"; "aaa"] (fun buf -> match%sedlex buf with | (Rep ('a' .. 'c', 0 .. 2) as z), 'a' -> "z=" ^ sub z | _ -> "nomatch"); [%expect {| "aa" -> z="a" "aaa" -> z="aa" |}] (* Bounded repetition of a nullable body. [Rep (r, 0 .. 1)] unrolls to [r | ""], and [r] itself parses "" through its first alternative [Opt 'd'] before it parses "b" through its second. Leftmost-greedy takes that first parse in all four forms: x="", with the Star taking the 'b'. The consuming parse used to win; this test makes a change of tie-break visible. *) let%expect_test "rep_nullable_body" = let sub = Sedlexing.Latin1.of_submatch in let run inputs lex = List.iter (fun s -> Printf.printf "%-4S -> %s\n" s (lex (Sedlexing.Latin1.from_string s))) inputs in run ["b"] (fun buf -> match%sedlex buf with | (Rep ((Opt 'd' | 'b'), 0 .. 1) as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "b" -> x="" |}]; run ["b"] (fun buf -> match%sedlex buf with | ((Opt 'd' | 'b' | "") as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "b" -> x="" |}]; run ["b"] (fun buf -> match%sedlex buf with | (Rep ((Opt 'd' | 'b'), 1 .. 2) as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "b" -> x="" |}]; run ["bb"] (fun buf -> match%sedlex buf with | (Rep ((Opt 'd' | 'b'), 0 .. 2) as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "bb" -> x="" |}] (* Falling back after a longer match failed must give the sub-matches of the last accepting state, though [Sedlexing.backtrack] restores no cell. Each pattern accepts, goes on with a longer alternative that records the same sub-match again, then fails (first input) or succeeds (second). *) let%expect_test "fallback_keeps_submatches" = let sub x = Printf.sprintf "%S" (Sedlexing.Latin1.of_submatch x) in let run inputs lex = List.iter (fun s -> let buf = Sedlexing.Latin1.from_string s in let r = lex buf in Printf.printf "%-9S -> %S %s\n" s (Sedlexing.Latin1.lexeme buf) r) inputs in run ["azdcwX"; "azdcwd"; "adX"] (fun buf -> match%sedlex buf with | ("a" | "azdc"), (Star ('z' | 'w') as x), 'd' -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "azdcwX" -> "azd" x="z" "azdcwd" -> "azdcwd" x="w" "adX" -> "ad" x="" |}]; (* same with an or-pattern: the longer branch must not leak into the discriminator either *) run ["azdcwX"; "azdcwwe"] (fun buf -> match%sedlex buf with | "a", (Star 'z' as x), 'd' | "azdc", (Plus 'w' as x), 'e' -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "azdcwX" -> "azd" x="z" "azdcwwe" -> "azdcwwe" x="ww" |}]; (* two sub-matches, only the second one recorded again *) run ["abzcbwX"; "abzcbwc"] (fun buf -> match%sedlex buf with | (Plus 'a' as x), ("b" | "bzcb"), (Star ('z' | 'w') as y), 'c' -> "x=" ^ sub x ^ " y=" ^ sub y | _ -> "nomatch"); [%expect {| "abzcbwX" -> "abzc" x="a" y="z" "abzcbwc" -> "abzcbwc" x="a" y="w" |}] (* Transition operations record the position before the character read, but reading end of input does not advance: sub-matches that end where eof, or either eof or a character, is read must get their end right. *) let%expect_test "submatch_end_before_eof_or_char" = let sub x = Printf.sprintf "%S" (Sedlexing.Latin1.of_submatch x) in let run inputs lex = List.iter (fun s -> let buf = Sedlexing.Latin1.from_string s in let r = lex buf in Printf.printf "%-6S -> %S %s\n" s (Sedlexing.Latin1.lexeme buf) r) inputs in (* eof or 'b' *) run ["aa"; "aab"; "a"; "ab"] (fun buf -> match%sedlex buf with | (Plus 'a' as x), (eof | 'b') -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "aa" -> "aa" x="aa" "aab" -> "aab" x="aa" "a" -> "a" x="a" "ab" -> "ab" x="a" |}]; (* eof after an optional tail *) run ["aa"; "aac"; "aacc"] (fun buf -> match%sedlex buf with | (Plus 'a' as x), Star 'c', eof -> "x=" ^ sub x | _ -> "nomatch"); [%expect {| "aa" -> "aa" x="aa" "aac" -> "aac" x="aa" "aacc" -> "aacc" x="aa" |}] (* Rep (_, 1 .. 1) is a single-character regexp, so Compl/Sub/Intersect accept it: it normalizes to the bare Chars node. *) let%expect_test "rep_1_1_char_ops" = let buf = Sedlexing.Utf8.from_string "b" in (match%sedlex buf with | Compl (Rep ('a', 1 .. 1)) -> print_string "compl-ok\n" | _ -> assert false); [%expect {| compl-ok |}]; let buf = Sedlexing.Utf8.from_string "b" in (match%sedlex buf with | Sub (any, Rep ('a', 1 .. 1)) -> print_string "sub-ok\n" | _ -> assert false); [%expect {| sub-ok |}]; let buf = Sedlexing.Utf8.from_string "a" in (match%sedlex buf with | Intersect ('a' .. 'c', Rep ('a', 1 .. 1)) -> print_string "inter-ok\n" | _ -> assert false); [%expect {| inter-ok |}] (* A nullable-only rule makes DFA state 0 an accepting sink; the generated code must not emit an empty `let rec` or call an undefined state function. [""] matches the zero-length prefix at position 0 regardless of input. *) let%expect_test "empty_pattern" = let lex buf = match%sedlex buf with "" -> "empty" | _ -> "other" in Printf.printf "%s\n" (lex (Sedlexing.Utf8.from_string "")); Printf.printf "%s\n" (lex (Sedlexing.Utf8.from_string "abc")); [%expect {| empty empty |}] (* Greedy repetition guards: they pin the disambiguation rules that the determinization must preserve. *) (* Opt is greedy (consume-first) and agrees with Rep (_, 0 .. 1): when the empty and the consuming parse have the same total length, the leading optional takes the character and the trailing Star stays empty. *) let%expect_test "opt_greedy" = let buf = Sedlexing.Utf8.from_string "a" in (match%sedlex buf with | Opt 'a', (Star 'a' as x) -> Printf.printf "opt x=%S\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| opt x="" |}]; let buf = Sedlexing.Utf8.from_string "a" in (match%sedlex buf with | Rep ('a', 0 .. 1), (Star 'a' as x) -> Printf.printf "rep x=%S\n" (Sedlexing.Utf8.of_submatch x) | _ -> assert false); [%expect {| rep x="" |}] (* [Plus r] is [r, Star r]: after a first iteration that consumed nothing, a second (consuming) iteration still has priority over leaving the loop, exactly as for [Star]. The capture must be greedy in all three forms. *) let%expect_test "plus_nullable_first_alternative" = let sub = Sedlexing.Latin1.of_submatch in let inputs = ["b"; "bb"; "db"] in let run lex = List.iter (fun s -> Printf.printf "%-4S -> %s\n" s (lex (Sedlexing.Latin1.from_string s))) inputs in run (fun buf -> match%sedlex buf with | (Plus (Opt 'd' | 'b') as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "b" -> x="b" "bb" -> x="bb" "db" -> x="db" |}]; run (fun buf -> match%sedlex buf with | (Star (Opt 'd' | 'b') as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "b" -> x="b" "bb" -> x="bb" "db" -> x="db" |}]; run (fun buf -> match%sedlex buf with | (((Opt 'd' | 'b'), Star (Opt 'd' | 'b')) as x), Star 'b' -> Printf.sprintf "x=%S" (sub x) | _ -> "nomatch"); [%expect {| "b" -> x="b" "bb" -> x="bb" "db" -> x="db" |}] let%expect_test "latin1_sub_lexeme" = let buf = Sedlexing.Latin1.from_string "h\233llo wor\255d" in let sub pos len = match Sedlexing.Latin1.sub_lexeme buf pos len with | s -> Printf.printf "%d %d -> %S\n" pos len s | exception Invalid_argument _ -> Printf.printf "%d %d -> Invalid_argument\n" pos len in (match%sedlex buf with | Plus (Compl ' ') -> Printf.printf "%S\n" (Sedlexing.Latin1.lexeme buf); sub 0 5; sub 1 3; sub 5 0; sub 2 (-1); sub (-1) 2; (* past the lexeme, inside the buffer *) sub 0 6; sub 4 2; sub 0 100 | _ -> assert false); [%expect {| "h\233llo" 0 5 -> "h\233llo" 1 3 -> "\233ll" 5 0 -> "" 2 -1 -> Invalid_argument -1 2 -> Invalid_argument 0 6 -> Invalid_argument 4 2 -> Invalid_argument 0 100 -> Invalid_argument |}]; (* A code point outside Latin1 *) let buf = Sedlexing.Utf8.from_string "a\xe2\x82\xacb" in (match%sedlex buf with | Plus any -> ( match Sedlexing.Latin1.lexeme buf with | s -> Printf.printf "%S\n" s | exception Sedlexing.InvalidCodepoint c -> Printf.printf "U+%04X\n" c ) | _ -> assert false); [%expect {| U+20AC |}] (* [Utf8.sub_lexeme] and [Utf16.sub_lexeme] against the encoders of the standard library, on every sub-range of a lexeme that mixes all the encoding lengths. *) let%expect_test "utf_sub_lexeme_encoding" = let cps = [| 0x41; 0x00; 0x7a; 0x7F; 0x80; 0x7FF; 0x800; 0xD7FF; 0xE000; 0xFEFF; 0xFFFD; 0xFFFF; 0x10000; 0x1F600; 0x10FFFF; 0x62; 0x63; |] in let a = Array.map Uchar.of_int cps in let n = Array.length a in let buf = Sedlexing.from_uchar_array a in (match%sedlex buf with Plus any -> () | _ -> assert false); assert (Sedlexing.lexeme_length buf = n); let reference add ~bom pos len = let b = Buffer.create 16 in if bom then add b (Uchar.of_int 0xFEFF); for i = pos to pos + len - 1 do add b a.(i) done; Buffer.contents b in let errors = ref 0 and checked = ref 0 in let check what got expected = incr checked; if got <> expected then begin incr errors; Printf.printf "%s: got %S, expected %S\n" what got expected end in for pos = 0 to n do for len = 0 to n - pos do check (Printf.sprintf "utf8 %d %d" pos len) (Sedlexing.Utf8.sub_lexeme buf pos len) (reference Buffer.add_utf_8_uchar ~bom:false pos len); List.iter (fun (name, bo, add) -> List.iter (fun bom -> check (Printf.sprintf "utf16%s %d %d bom=%b" name pos len bom) (Sedlexing.Utf16.sub_lexeme buf pos len bo bom) (reference add ~bom pos len)) [false; true]) [ ("le", Sedlexing.Utf16.Little_endian, Buffer.add_utf_16le_uchar); ("be", Sedlexing.Utf16.Big_endian, Buffer.add_utf_16be_uchar); ] done done; check "utf8 lexeme" (Sedlexing.Utf8.lexeme buf) (reference Buffer.add_utf_8_uchar ~bom:false 0 n); check "utf16 lexeme" (Sedlexing.Utf16.lexeme buf Sedlexing.Utf16.Big_endian true) (reference Buffer.add_utf_16be_uchar ~bom:true 0 n); Printf.printf "%d checks, %d errors\n" !checked !errors; [%expect {| 857 checks, 0 errors |}] let%expect_test "utf_sub_lexeme_range" = let buf = Sedlexing.Utf8.from_string "h\xc3\xa9llo w\xc3\xb6rld" in let show name f pos len = match f pos len with | s -> Printf.printf "%s %d %d -> %S\n" name pos len s | exception Invalid_argument _ -> Printf.printf "%s %d %d -> Invalid_argument\n" name pos len in let uchars = show "uchars" (fun pos len -> Sedlexing.sub_lexeme buf pos len |> Array.map (fun c -> Printf.sprintf "%X" (Uchar.to_int c)) |> Array.to_list |> String.concat " ") in let utf8 = show "utf8" (Sedlexing.Utf8.sub_lexeme buf) in let utf16 = show "utf16" (fun pos len -> Sedlexing.Utf16.sub_lexeme buf pos len Sedlexing.Utf16.Big_endian true) in (match%sedlex buf with | Plus (Compl ' ') -> List.iter (fun f -> f 0 5; f 1 3; f 5 0; f 2 (-1); f (-1) 2; (* past the lexeme, inside the buffer *) f 0 6; f 4 2; f 0 100) [uchars; utf8; utf16] | _ -> assert false); [%expect {| uchars 0 5 -> "68 E9 6C 6C 6F" uchars 1 3 -> "E9 6C 6C" uchars 5 0 -> "" uchars 2 -1 -> Invalid_argument uchars -1 2 -> Invalid_argument uchars 0 6 -> Invalid_argument uchars 4 2 -> Invalid_argument uchars 0 100 -> Invalid_argument utf8 0 5 -> "h\195\169llo" utf8 1 3 -> "\195\169ll" utf8 5 0 -> "" utf8 2 -1 -> Invalid_argument utf8 -1 2 -> Invalid_argument utf8 0 6 -> Invalid_argument utf8 4 2 -> Invalid_argument utf8 0 100 -> Invalid_argument utf16 0 5 -> "\254\255\000h\000\233\000l\000l\000o" utf16 1 3 -> "\254\255\000\233\000l\000l" utf16 5 0 -> "\254\255" utf16 2 -1 -> Invalid_argument utf16 -1 2 -> Invalid_argument utf16 0 6 -> Invalid_argument utf16 4 2 -> Invalid_argument utf16 0 100 -> Invalid_argument |}] let%expect_test "utf_of_submatch_non_ascii" = let buf = Sedlexing.Utf8.from_string "h\xc3\xa9llo w\xc3\xb6rld\xe2\x82\xac\xf0\x9f\x98\x80" in (match%sedlex buf with | (Plus (Compl ' ') as x), ' ', (Plus any as y) -> Printf.printf "utf8: %S %S\n" (Sedlexing.Utf8.of_submatch x) (Sedlexing.Utf8.of_submatch y); Printf.printf "utf16le: %S\n" (Sedlexing.Utf16.of_submatch y Sedlexing.Utf16.Little_endian false); Printf.printf "utf16be+bom: %S\n" (Sedlexing.Utf16.of_submatch x Sedlexing.Utf16.Big_endian true) | _ -> assert false); [%expect {| utf8: "h\195\169llo" "w\195\182rld\226\130\172\240\159\152\128" utf16le: "w\000\246\000r\000l\000d\000\172 =\216\000\222" utf16be+bom: "\254\255\000h\000\233\000l\000l\000o" |}] (* Extraction from tokens that do not start the buffer, read through a generator so that the buffer is refilled and compacted several times. The reference is built from the code points of [Sedlexing.lexeme]. *) let%expect_test "sub_lexeme_later_tokens" = let run name ~latin1 cps = let text = Array.init 3000 (fun i -> Uchar.of_int (if i mod 7 = 3 then 0x20 else cps.(i * 5 mod Array.length cps))) in let buf = let i = ref 0 in Sedlexing.from_gen (fun () -> if !i >= Array.length text then None else ( incr i; Some text.(!i - 1))) in let encode add a = let b = Buffer.create 16 in Array.iter (add b) a; Buffer.contents b in let tokens = ref 0 and errors = ref 0 in let check what got expected = if got <> expected then begin incr errors; if !errors <= 5 then Printf.printf "%s, token %d, %s: got %S, expected %S\n" name !tokens what got expected end in let token () = incr tokens; let a = Sedlexing.lexeme buf in let n = Array.length a in (* the whole lexeme, then without its first and last code points *) List.iter (fun (pos, len) -> let a = Array.sub a pos len in check "utf8" (Sedlexing.Utf8.sub_lexeme buf pos len) (encode Buffer.add_utf_8_uchar a); check "utf16le" (Sedlexing.Utf16.sub_lexeme buf pos len Sedlexing.Utf16.Little_endian false) (encode Buffer.add_utf_16le_uchar a); check "utf16be" (Sedlexing.Utf16.sub_lexeme buf pos len Sedlexing.Utf16.Big_endian true) ("\xfe\xff" ^ encode Buffer.add_utf_16be_uchar a); if latin1 then check "latin1" (Sedlexing.Latin1.sub_lexeme buf pos len) (encode (fun b u -> Buffer.add_char b (Uchar.to_char u)) a)) [(0, n); (1, n - 1); (0, n - 1)] in let rec loop () = match%sedlex buf with | Plus (Compl ' ') | ' ' -> token (); loop () | eof -> () | _ -> assert false in loop (); Printf.printf "%s: %d tokens, %d errors\n" name !tokens !errors in run "all widths" ~latin1:false [| 0x61; 0xE9; 0x20AC; 0x1F600; 0x62; 0x7F; 0x80; 0x7FF |]; run "latin1" ~latin1:true [| 0x61; 0xE9; 0xFF; 0x62; 0x7F; 0x80 |]; [%expect {| all widths: 858 tokens, 0 errors latin1: 858 tokens, 0 errors |}] (* The memory cells are not cleared between tokens: each action must only see the cells written by its own match, whatever earlier tokens and other rules sharing the lexbuf left behind. *) let%expect_test "stale_cells_between_tokens" = let sub = Sedlexing.Utf8.of_submatch in let rec other buf = match%sedlex buf with | (Plus 'x' as a), (Plus 'y' as b), (Plus 'z' as c), Plus 'x' -> Printf.printf "other: a=%S b=%S c=%S\n" (sub a) (sub b) (sub c); token buf | _ -> token buf and token buf = match%sedlex buf with | (Plus 'a' as k), '=', (Plus 'b' as v) | (Plus 'b' as v), ':', (Plus 'a' as k) -> Printf.printf "pair: k=%S v=%S\n" (sub k) (sub v); other buf | Plus 'a', (Plus 'c' as v) | (Plus 'c' as v), Plus 'b' -> Printf.printf "single: v=%S\n" (sub v); other buf | ' ' -> other buf | eof -> () | _ -> assert false in token (Sedlexing.Utf8.from_string "aaa=b bb:a a=bbb xxyzzzx b:aa acc cb aaaac xyyzx ccccbb"); [%expect {| pair: k="aaa" v="b" pair: k="a" v="bb" pair: k="a" v="bbb" other: a="xx" b="y" c="zzz" pair: k="aa" v="b" single: v="cc" single: v="c" single: v="c" other: a="x" b="yy" c="z" single: v="cccc" |}] (* Same as above with the cells overwritten before every token, by the value they used to be cleared with (-1), by plausible positions and by encoded discriminator values: the submatches must not depend on it. The input is read through a generator and is long enough for the buffer to be refilled, which shifts the positions left in the cells. *) let%expect_test "poisoned_cells" = let sub = Sedlexing.Utf8.of_submatch in let digit = [%sedlex.regexp? '0' .. '9'] in let rule_a buf = match%sedlex buf with (* nested or-patterns *) | ('a', (Plus 'x' as v) | 'b', (Plus 'y' as v)), ';' | 'c', ((Plus 'z' as v) | (Plus 'w' as v)), ';' -> Some ("nested " ^ sub v) (* a loop before an overlapping capture *) | Star 'a', (Plus ('a' | 'b') as u), '!' -> Some ("loop " ^ sub u) | (Plus digit as n), '.', (Plus digit as f) -> Some ("float " ^ sub n ^ " " ^ sub f) (* a capture that may end at the end of input *) | '#', (Star (Compl ' ') as c), (' ' | eof) -> Some ("comment " ^ sub c) | ' ' -> Some "space" | eof -> None | any -> Some ("any " ^ Sedlexing.Utf8.lexeme buf) | _ -> assert false in let rule_b buf = match%sedlex buf with | (Plus 'k' as k), '=', (Plus 'v' as v) | (Plus 'v' as v), ':', (Plus 'k' as k) -> Some ("pair " ^ sub k ^ " " ^ sub v) | (Plus digit as n), Opt ('e', Plus digit) -> Some ("int " ^ sub n) | ' ' -> Some "space" | eof -> None | any -> Some ("any " ^ Sedlexing.Utf8.lexeme buf) | _ -> assert false in let input = let words = [| "axx;"; "kk=v"; "byyy;"; "v:k"; "czz;"; "12"; "cw;"; "vvv:kk"; "aab!"; "k=vv"; "ab!"; "7"; "bb!"; "kkk=vvv"; "12.5"; "v:kkk"; "3.14159"; "42"; "#note"; "k=v"; "x"; "y"; |] in let b = Buffer.create 4096 in for i = 0 to 399 do Buffer.add_string b words.(i * 7 mod Array.length words); Buffer.add_char b ' ' done; Buffer.add_string b "#end"; Buffer.contents b in let run poison = let buf = Sedlexing.Utf8.from_gen (gen_from_string input) in Sedlexing.__private__ensure_mem buf 16; let out = Buffer.create 4096 in let seed = ref 42 in let rec loop i = (match poison with | None -> () | Some v -> for c = 0 to Sedlexing.__private__num_mem_cells buf - 1 do Sedlexing.__private__mem_set buf c v done); (* two rules with different cells, picked by a fixed pseudo-random sequence so that every word meets both *) seed := ((!seed * 1103515245) + 12345) land 0x3FFFFFFF; match if (!seed lsr 16) land 1 = 0 then rule_a buf else rule_b buf with | None -> i | Some s -> Buffer.add_string out s; Buffer.add_char out '\n'; loop (i + 1) in let n = loop 0 in (n, Buffer.contents out) in let n, reference = run None in Printf.printf "%d characters, %d tokens\n" (String.length input) n; (* how many tokens of each kind, and a few of them *) let lines = String.split_on_char '\n' reference in let kind l = List.hd (String.split_on_char ' ' l) in List.iter (fun k -> let ls = List.filter (fun l -> kind l = k) lines in Printf.printf "%-8s %4d e.g. %s\n" k (List.length ls) (String.concat ", " (List.sort_uniq compare ls |> List.filteri (fun i _ -> i < 4)))) ["nested"; "loop"; "float"; "comment"; "pair"; "int"]; List.iter (fun v -> Printf.printf "cells set to %d: %s\n" v (if run (Some v) = (n, reference) then "same" else "DIFFERENT")) [-1; 0; 1; 3; 600; 1_000_003; -2; -3; -4; -5]; [%expect {| 1877 characters, 1162 tokens nested 39 e.g. nested w, nested xx, nested yyy, nested zz loop 43 e.g. loop b, loop bb float 24 e.g. float 12 5, float 3 14159 comment 11 e.g. comment end, comment note pair 83 e.g. pair k v, pair k vv, pair k vvv, pair kk v int 61 e.g. int 12, int 14159, int 2, int 3 cells set to -1: same cells set to 0: same cells set to 1: same cells set to 3: same cells set to 600: same cells set to 1000003: same cells set to -2: same cells set to -3: same cells set to -4: same cells set to -5: same |}] sedlex-3.8.1/test/codegen/000077500000000000000000000000001526113672700154045ustar00rootroot00000000000000sedlex-3.8.1/test/codegen/dune000066400000000000000000000002421526113672700162600ustar00rootroot00000000000000(library (name sedlex_gen_test) (libraries sedlex) (inline_tests) (enabled_if (>= %{ocaml_version} 4.14)) (preprocess (pps ppx_sedlex_test ppx_expect))) sedlex-3.8.1/test/codegen/render_dots.sh000077500000000000000000000032331526113672700202540ustar00rootroot00000000000000#!/usr/bin/env bash # Extract DOT graphs from expect test files and render them as SVG images. # # Usage: ./render_dots.sh [output_dir] # output_dir defaults to ./dot_images set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" OUT_DIR="${1:-$SCRIPT_DIR/dot_images}" mkdir -p "$OUT_DIR" count=0 for ml_file in "$SCRIPT_DIR"/test_gen.ml "$SCRIPT_DIR"/test_realistic.ml; do [ -f "$ml_file" ] || continue base="$(basename "$ml_file" .ml)" # State machine: scan for test names and DOT blocks test_name="" in_dot=false dot_buf="" while IFS= read -r line; do # Detect test name from let%expect_test "..." if [[ "$line" =~ let%expect_test\ \"(.+)\" ]]; then test_name="${BASH_REMATCH[1]}" continue fi # Detect start of DOT block if [[ "$in_dot" == false && "$line" =~ ^[[:space:]]*DOT: ]]; then in_dot=true dot_buf="" continue fi # Detect end of DOT block (CODE: line or closing |}) if [[ "$in_dot" == true ]]; then if [[ "$line" =~ ^[[:space:]]*CODE: ]] || [[ "$line" =~ ^\|?\} ]]; then in_dot=false if [ -n "$dot_buf" ] && [ -n "$test_name" ]; then # Sanitize test name for filename safe_name="$(echo "$test_name" | sed 's/[^A-Za-z0-9_-]/_/g')" out_file="$OUT_DIR/${base}__${safe_name}.svg" echo "$dot_buf" | dot -Tsvg -o "$out_file" 2>/dev/null && { echo " $out_file" count=$((count + 1)) } || echo " WARN: failed to render $safe_name" >&2 fi dot_buf="" continue fi dot_buf="$dot_buf $line" fi done < "$ml_file" done echo "Rendered $count images in $OUT_DIR" sedlex-3.8.1/test/codegen/test_errors.ml000066400000000000000000000423161526113672700203170ustar00rootroot00000000000000(* Error tests for `as` bindings *) let%expect_test "error: as inside Star" = [%compile_error [%sedlex match buf with Star ('a' as x) -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 33-43: | [%sedlex match buf with Star ('a' as x) -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Star |}] let%expect_test "error: as inside Plus" = [%compile_error [%sedlex match buf with Plus ('a' as x) -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 33-43: | [%sedlex match buf with Plus ('a' as x) -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Plus |}] let%expect_test "error: as inside Opt" = [%compile_error [%sedlex match buf with Opt ('a' as x) -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 32-42: | [%sedlex match buf with Opt ('a' as x) -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Opt |}] let%expect_test "error: as inside Rep" = [%compile_error [%sedlex match buf with Rep ((('a' as x), 'b'), 2 .. 3) -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 26-43: | match buf with Rep ((('a' as x), 'b'), 2 .. 3) -> ignore x | _ -> ()]]; ^^^^^^^^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Rep |}] let%expect_test "error: as inside Compl" = [%compile_error [%sedlex match buf with Compl ('a' as x) -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 34-44: | [%sedlex match buf with Compl ('a' as x) -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Compl |}] let%expect_test "error: as inside Sub" = [%compile_error [%sedlex match buf with Sub (('a' as x), 'b') -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 33-43: | [%sedlex match buf with Sub (('a' as x), 'b') -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Sub |}] let%expect_test "error: as inside Intersect" = [%compile_error [%sedlex match buf with Intersect (('a' as x), 'b') -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 39-49: | [%sedlex match buf with Intersect (('a' as x), 'b') -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' bindings are not supported inside Intersect |}] let%expect_test "error: as shadows inner binding" = [%compile_error [%sedlex match buf with (('a' as x), 'b') as x -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 49-50: | [%sedlex match buf with (('a' as x), 'b') as x -> ignore x | _ -> ()]]; ^ Error: Sedlex: 'as' binding 'x' shadows an inner binding of the same name |}] let%expect_test "error: duplicate as binding in sequence" = [%compile_error [%sedlex match buf with ('a' as x), 'b', ('c' as x) -> ignore x | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 45-55: | [%sedlex match buf with ('a' as x), 'b', ('c' as x) -> ignore x | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: 'as' binding 'x' is bound in multiple positions |}] let%expect_test "error: different names in or-pattern" = [%compile_error [%sedlex match buf with ('a' as x) | ('b' as y) -> ignore (x, y) | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 21-44: | match buf with ('a' as x) | ('b' as y) -> ignore (x, y) | _ -> ()]]; ^^^^^^^^^^^^^^^^^^^^^^^ Error: Sedlex: all branches of '|' must bind the same names with 'as' |}] (* Error tests for Sub/Intersect/Compl on multi-char regexps *) let%expect_test "error: Sub on multi-char regexp" = [%compile_error [%sedlex match buf with Sub ("ab", 'a') -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-57: | [%compile_error [%sedlex match buf with Sub ("ab", 'a') -> () | _ -> ()]]; ^^^^^^^^^^^^^^^ Error: Sedlex: the Sub operator can only applied to single-character length regexps |}] let%expect_test "error: Intersect on multi-char regexp" = [%compile_error [%sedlex match buf with Intersect ("ab", 'a') -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 28-49: | [%sedlex match buf with Intersect ("ab", 'a') -> () | _ -> ()]]; ^^^^^^^^^^^^^^^^^^^^^ Error: Sedlex: the Intersect operator can only applied to single-character length regexps |}] let%expect_test "error: Compl on multi-char regexp" = [%compile_error [%sedlex match buf with Compl "ab" -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-52: | [%compile_error [%sedlex match buf with Compl "ab" -> () | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: the Compl operator can only applied to a single-character length regexp |}] let%expect_test "error: Sub with one argument" = [%compile_error [%sedlex match buf with Sub 'a' -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-49: | [%compile_error [%sedlex match buf with Sub 'a' -> () | _ -> ()]]; ^^^^^^^ Error: Sedlex: the Sub operator requires two arguments, like Sub(a,b) |}] let%expect_test "error: Intersect with one argument" = [%compile_error [%sedlex match buf with Intersect 'a' -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-55: | [%compile_error [%sedlex match buf with Intersect 'a' -> () | _ -> ()]]; ^^^^^^^^^^^^^ Error: Sedlex: the Intersect operator requires two arguments, like Intersect(a,b) |}] (* Error tests for Rep *) let%expect_test "error: Rep invalid range" = [%compile_error [%sedlex match buf with Rep ('a', 5 .. 2) -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-59: | [%compile_error [%sedlex match buf with Rep ('a', 5 .. 2) -> () | _ -> ()]]; ^^^^^^^^^^^^^^^^^ Error: Sedlex: Invalid range for Rep operator |}] let%expect_test "error: Rep with non-integer" = [%compile_error [%sedlex match buf with Rep ('a', 'b' .. 'c') -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 28-49: | [%sedlex match buf with Rep ('a', 'b' .. 'c') -> () | _ -> ()]]; ^^^^^^^^^^^^^^^^^^^^^ Error: Sedlex: Rep must take an integer constant or interval |}] let%expect_test "error: Rep with one argument" = [%compile_error [%sedlex match buf with Rep 'a' -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-49: | [%compile_error [%sedlex match buf with Rep 'a' -> () | _ -> ()]]; ^^^^^^^ Error: Sedlex: the Rep operator takes 2 arguments |}] (* Error tests for Compl/Chars *) let%expect_test "error: Compl without argument" = [%compile_error [%sedlex match buf with Compl -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-47: | [%compile_error [%sedlex match buf with Compl -> () | _ -> ()]]; ^^^^^ Error: Sedlex: the Compl operator requires an argument |}] let%expect_test "error: Chars with non-string" = [%compile_error [%sedlex match buf with Chars 42 -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-50: | [%compile_error [%sedlex match buf with Chars 42 -> () | _ -> ()]]; ^^^^^^^^ Error: Sedlex: the Chars operator requires a string argument |}] (* Error tests for unbound regexp and invalid patterns *) let%expect_test "error: unbound regexp" = [%compile_error [%sedlex match buf with nonexistent_regexp -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-60: | [%compile_error [%sedlex match buf with nonexistent_regexp -> () | _ -> ()]]; ^^^^^^^^^^^^^^^^^^ Error: Sedlex: unbound regexp nonexistent_regexp |}] let%expect_test "error: invalid pattern" = [%compile_error [%sedlex match buf with Some 'a' -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-50: | [%compile_error [%sedlex match buf with Some 'a' -> () | _ -> ()]]; ^^^^^^^^ Error: Sedlex: unknown sedlex operator Some |}] let%expect_test "error: invalid interval type" = [%compile_error [%sedlex match buf with 1.0 .. 2.0 -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-52: | [%compile_error [%sedlex match buf with 1.0 .. 2.0 -> () | _ -> ()]]; ^^^^^^^^^^ Error: Sedlex: this pattern is not a valid interval regexp |}] let%expect_test "error: invalid constant pattern" = [%compile_error [%sedlex match buf with 1.0 -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-45: | [%compile_error [%sedlex match buf with 1.0 -> () | _ -> ()]]; ^^^ Error: Sedlex: this pattern is not a valid regexp |}] (* Error tests for match structure *) let%expect_test "error: missing catch-all" = [%compile_error [%sedlex match buf with 'a' -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-45: | [%compile_error [%sedlex match buf with 'a' -> ()]]; ^^^ Error: Sedlex: the last branch must be a catch-all error case |}] let%expect_test "error: when guard" = [%compile_error [%sedlex match buf with 'a' when true -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 51-55: | [%compile_error [%sedlex match buf with 'a' when true -> () | _ -> ()]]; ^^^^ Error: Sedlex: 'when' guards are not supported |}] let%expect_test "error: matched expression must be a single identifier" = [%compile_error [%sedlex match foo bar with 'a' -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 33-40: | [%compile_error [%sedlex match foo bar with 'a' -> () | _ -> ()]]; ^^^^^^^ Error: Sedlex: the matched expression must be a single identifier |}] (* Error tests for malformed strings *) let%expect_test "error: malformed ASCII string" = [%compile_error [%sedlex match buf with "\x80" -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-48: | [%compile_error [%sedlex match buf with "\x80" -> () | _ -> ()]]; ^^^^^^ Error: Sedlex: Malformed ASCII string |}] let%expect_test "error: malformed UTF-8 string" = [%compile_error [%sedlex match buf with Utf8 "\x80" -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 47-53: | [%compile_error [%sedlex match buf with Utf8 "\x80" -> () | _ -> ()]]; ^^^^^^ Error: Sedlex: Malformed UTF-8 string |}] let%expect_test "error: non-ASCII char interval" = [%compile_error [%sedlex match buf with '\x80' .. '\xff' -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-58: | [%compile_error [%sedlex match buf with '\x80' .. '\xff' -> () | _ -> ()]]; ^^^^^^^^^^^^^^^^ Error: Sedlex: this pattern is not a valid ASCII interval regexp |}] (* Error tests for sedlex extension misuse *) let%expect_test "error: sedlex not on match expression" = [%compile_error [%sedlex 42]]; [%expect {| File "test/codegen/test_errors.ml", characters 18-30: | [%compile_error [%sedlex 42]]; ^^^^^^^^^^^^ Error: Sedlex: the %sedlex extension is only recognized on match expressions |}] (* Error tests for bare operators (missing argument) *) let%expect_test "error: Star without argument" = [%compile_error [%sedlex match buf with Star -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-46: | [%compile_error [%sedlex match buf with Star -> () | _ -> ()]]; ^^^^ Error: Sedlex: the Star operator requires an argument |}] let%expect_test "error: Plus without argument" = [%compile_error [%sedlex match buf with Plus -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-46: | [%compile_error [%sedlex match buf with Plus -> () | _ -> ()]]; ^^^^ Error: Sedlex: the Plus operator requires an argument |}] let%expect_test "error: Opt without argument" = [%compile_error [%sedlex match buf with Opt -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-45: | [%compile_error [%sedlex match buf with Opt -> () | _ -> ()]]; ^^^ Error: Sedlex: the Opt operator requires an argument |}] (* Error tests for regexp definitions *) let%expect_test "error: as in regexp definition" = [%compile_error let dummy = [%sedlex.regexp? 'a' as x] in ignore dummy]; [%expect {| File "test/codegen/test_errors.ml", characters 33-41: | let dummy = [%sedlex.regexp? 'a' as x] in ^^^^^^^^ Error: Sedlex: 'as' bindings are not allowed in regexp definitions |}] (* ------------------------------------------------------------------------ *) (* Edge cases pinned to the current behavior. *) (* Rep (c, 1 .. 1) is a single-character regexp (it normalizes to c), so Compl, Sub and Intersect accept it. *) let%expect_test "no error: Rep 1..1 under Compl" = [%compile_error [%sedlex match buf with Compl (Rep ('a', 1 .. 1)) -> () | _ -> ()]]; [%expect {| NO ERROR |}] let%expect_test "no error: Rep 1..1 under Sub" = [%compile_error [%sedlex match buf with Sub (any, Rep ('a', 1 .. 1)) -> () | _ -> ()]]; [%expect {| NO ERROR |}] let%expect_test "no error: Rep 1..1 under Intersect" = [%compile_error [%sedlex match buf with Intersect ('a' .. 'c', Rep ('a', 1 .. 1)) -> () | _ -> ()]]; [%expect {| NO ERROR |}] (* An empty character set can never match, so it is rejected where it is formed: [Chars ""], [Compl any], and [Sub] / [Intersect] of coinciding classes. *) let%expect_test "error: empty Chars" = [%compile_error [%sedlex match buf with 'a', Chars "" -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 47-55: | [%compile_error [%sedlex match buf with 'a', Chars "" -> () | _ -> ()]]; ^^^^^^^^ Error: Sedlex: empty character set: this pattern can never match |}] let%expect_test "error: Compl any" = [%compile_error [%sedlex match buf with Compl any -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-51: | [%compile_error [%sedlex match buf with Compl any -> () | _ -> ()]]; ^^^^^^^^^ Error: Sedlex: empty character set: this pattern can never match |}] let%expect_test "error: Sub of coinciding classes" = [%compile_error [%sedlex match buf with Sub ('a', 'a') -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 42-56: | [%compile_error [%sedlex match buf with Sub ('a', 'a') -> () | _ -> ()]]; ^^^^^^^^^^^^^^ Error: Sedlex: empty character set: this pattern can never match |}] let%expect_test "error: Intersect of disjoint classes" = [%compile_error [%sedlex match buf with Intersect ('a', 'b') -> () | _ -> ()]]; [%expect {| File "test/codegen/test_errors.ml", characters 28-48: | [%sedlex match buf with Intersect ('a', 'b') -> () | _ -> ()]]; ^^^^^^^^^^^^^^^^^^^^ Error: Sedlex: empty character set: this pattern can never match |}] sedlex-3.8.1/test/codegen/test_gen.ml000066400000000000000000001731511526113672700175560ustar00rootroot00000000000000let%expect_test "simple string match" = (match%sedlex_test buf with "ab" | "de" -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state3 [label="'d'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2\n[rule 0]", shape=doublecircle]; state3 [label="3"]; state3 -> state2 [label="'e'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> () | _ -> () |}] let%expect_test "character class" = (match%sedlex_test buf with Plus 'a' .. 'z' -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'-'z'"]; state1 [label="1\n[rule 0]", shape=doublecircle]; state1 -> state1 [label="'a'-'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = Sedlexing.mark buf 0; (match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> () | _ -> () |}] let%expect_test "multi-rule" = (match%sedlex_test buf with | "ab" -> () | "de" -> () | Plus '0' .. '9' -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'0'-'9'"]; state0 -> state2 [label="'a'"]; state0 -> state4 [label="'d'"]; state1 [label="1\n[rule 2]", shape=doublecircle]; state1 -> state1 [label="'0'-'9'"]; state2 [label="2"]; state2 -> state3 [label="'b'"]; state3 [label="3\n[rule 0]", shape=doublecircle]; state4 [label="4"]; state4 -> state5 [label="'e'"]; state5 [label="5\n[rule 1]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_2 buf | 2 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = Sedlexing.mark buf 2; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> 1 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> () | 1 -> () | 2 -> () | _ -> () |}] let%expect_test "as binding: simple" = (match%sedlex_test buf with 'a', ('b' as x), 'c' -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2"]; state2 -> state3 [label="'c'"]; state3 [label="3\n[rule 0]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 1 in let __e = (Sedlexing.lexeme_length buf) - 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Counterpart of "as binding: simple" where no boundary is statically known: variable-length context on both sides and a variable-length capture force both a start and an end tag. *) let%expect_test "as binding: simple, no static elimination" = (match%sedlex_test buf with | Plus 'a', (Plus 'b' as x), Plus 'c' -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t2}"]; state2 [label="2"]; state2 -> state2 [label="'b'"]; state2 -> state3 [label="'c' {t3}"]; state3 [label="3\n[rule 0]\n{t1<-t3,t0<-t2}", shape=doublecircle]; state3 -> state3 [label="'c'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 2; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 3; __sedlex_state_3 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = Sedlexing.__private__copy_mem buf 1 3; Sedlexing.__private__copy_mem buf 0 2; Sedlexing.mark buf 0; (match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 4; __sedlex_state_0 buf with | 0 -> let x = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.__private__mem_pos buf 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] let%expect_test "as binding: whole-match shortcut" = (match%sedlex_test buf with Plus 'a' .. 'z' as x -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'-'z'"]; state1 [label="1\n[rule 0]", shape=doublecircle]; state1 -> state1 [label="'a'-'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = Sedlexing.mark buf 0; (match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] let%expect_test "as binding: multiple bindings" = (match%sedlex_test buf with | ('a' as x), ('b' as y), 'c' -> ignore (x, y) | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2"]; state2 -> state3 [label="'c'"]; state3 [label="3\n[rule 0]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 2 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let y = let __s = 1 in let __e = (Sedlexing.lexeme_length buf) - 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (x, y) | _ -> () |}] (* Counterpart of "as binding: multiple bindings" where lengths are not static: x needs an end tag, and y's start is anchored off that same tag (Tag + 1) via the left-context fallback in Seq lowering. *) let%expect_test "as binding: multiple bindings, no static elimination" = (match%sedlex_test buf with | (Plus 'a' as x), '-', (Plus 'b' as y) -> ignore (x, y) | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="'-' {t1}"]; state1 -> state1 [label="'a'"]; state2 [label="2"]; state2 -> state3 [label="'b'"]; state3 [label="3\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state3 -> state3 [label="'b'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_2 buf) | 1 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let y = let __s = (Sedlexing.__private__mem_pos buf 0) + 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (x, y) | _ -> () |}] (* Counterpart of "optim: discriminator elision" where the positions cannot be derived statically: each branch allocates a real position tag for x, and the branches get distinct discriminator values so extraction can pick the right tag at runtime. *) let%expect_test "as binding: or-pattern with discriminator, no static elimination" = (match%sedlex_test buf with | Plus 'a', (Plus 'b' as x) | (Plus 'b' as x), Plus 'a' -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state3 [label="'b'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t3}"]; state2 [label="2\n[rule 0]\n{d2=0,t0<-t3}", shape=doublecircle]; state2 -> state2 [label="'b'"]; state3 [label="3"]; state3 -> state4 [label="'a' {t4}"]; state3 -> state3 [label="'b'"]; state4 [label="4\n[rule 0]\n{d2=1,t1<-t4}", shape=doublecircle]; state4 -> state4 [label="'a'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 3; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = Sedlexing.__private__set_mem_value buf 2 0; Sedlexing.__private__copy_mem buf 0 3; Sedlexing.mark buf 0; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_3 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_prev_pos buf 4; __sedlex_state_4 buf) | 1 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = Sedlexing.__private__set_mem_value buf 2 1; Sedlexing.__private__copy_mem buf 1 4; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 5; __sedlex_state_0 buf with | 0 -> let x = if (Sedlexing.__private__mem_value buf 2) = 0 then let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } else (let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) in ignore x | _ -> () |}] let%expect_test "as binding: shared prefix or-pattern" = (match%sedlex_test buf with | ("abc" as x), "def" | "a", ("bcd" as x), "ey" -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2"]; state2 -> state3 [label="'c'"]; state3 [label="3"]; state3 -> state4 [label="'d'"]; state4 [label="4"]; state4 -> state5 [label="'e'"]; state5 [label="5"]; state5 -> state6 [label="'f'"]; state5 -> state7 [label="'y'"]; state6 [label="6\n[rule 0]\n{d0=0}", shape=doublecircle]; state7 [label="7\n[rule 0]\n{d0=1}", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = match __sedlex_partition_5 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_5 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_5 buf = match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_value buf 0 0; 0) | 1 -> (Sedlexing.__private__set_mem_value buf 0 1; 0) | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 1; __sedlex_state_0 buf with | 0 -> let x = if (Sedlexing.__private__mem_value buf 0) = 0 then let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 3 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } else (let __s = 1 in let __e = (Sedlexing.lexeme_length buf) - 2 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) in ignore x | _ -> () |}] (* One discriminator cell for the three branches. The first and third bind x at the same offsets and share value 0; only the second needs its own. *) let%expect_test "as binding: 3-way or reuses disc cell" = (match%sedlex_test buf with | ("ab" as x), "cd" | ("a" as x), "bce" | ("abc" as x), "df" -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2"]; state2 -> state3 [label="'c'"]; state3 [label="3"]; state3 -> state4 [label="'d'"]; state3 -> state6 [label="'e'"]; state4 [label="4\n[rule 0]\n{d0=0}", shape=doublecircle]; state4 -> state5 [label="'f'"]; state5 [label="5\n[rule 0]\n{d0=0}", shape=doublecircle]; state6 [label="6\n[rule 0]\n{d0=1}", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | 1 -> (Sedlexing.__private__set_mem_value buf 0 1; 0) | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = Sedlexing.__private__set_mem_value buf 0 0; Sedlexing.mark buf 0; (match __sedlex_partition_5 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_value buf 0 0; 0) | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 1; __sedlex_state_0 buf with | 0 -> let x = if (Sedlexing.__private__mem_value buf 0) = 0 then let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 2 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } else if (Sedlexing.__private__mem_value buf 0) = 1 then (let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 3 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) else (let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 2 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) in ignore x | _ -> () |}] let%expect_test "as binding: multi-rule" = (match%sedlex_test buf with | 'a', ('b' as x) -> ignore x | "cd" -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state3 [label="'c'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2\n[rule 0]", shape=doublecircle]; state3 [label="3"]; state3 -> state4 [label="'d'"]; state4 [label="4\n[rule 1]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> 1 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | 1 -> () | _ -> () |}] (* Counterpart of "as binding: multi-rule" where the capture's start is not statically known: the variable-length prefix forces a start tag, while the untagged second rule shares the DFA. *) let%expect_test "as binding: multi-rule, no static elimination" = (match%sedlex_test buf with | Plus 'a', (Plus 'b' as x) -> ignore x | "cd" -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state3 [label="'c'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t1}"]; state2 [label="2\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state2 -> state2 [label="'b'"]; state3 [label="3"]; state3 -> state4 [label="'d'"]; state4 [label="4\n[rule 1]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_3 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> 1 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | 1 -> () | _ -> () |}] let%expect_test "as binding: wrapping alternation" = (match%sedlex_test buf with 'x', (('a' | 'b') as y) -> ignore y | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'x'"]; state1 [label="1"]; state1 -> state2 [label="'a'-'b'"]; state2 [label="2\n[rule 0]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let y = let __s = 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore y | _ -> () |}] (* Counterpart of "as binding: wrapping alternation" where the wrapped alternation has branches of different lengths and a variable-length suffix follows: the capture's end cannot be computed statically and needs an end tag. *) let%expect_test "as binding: wrapping alternation, no static elimination" = (match%sedlex_test buf with | 'x', (("ab" | "c") as y), Plus 'z' -> ignore y | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'x'"]; state1 [label="1"]; state1 -> state2 [label="'a'"]; state1 -> state3 [label="'c'"]; state2 [label="2"]; state2 -> state3 [label="'b'"]; state3 [label="3"]; state3 -> state4 [label="'z' {t1}"]; state4 [label="4\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state4 -> state4 [label="'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | 1 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_4 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let y = let __s = 1 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore y | _ -> () |}] (* === Optimization tracking tests === Each test below targets a specific optimization. As optimizations are implemented, the expected output will improve (fewer tags, fewer set_mem calls, etc.). *) (* Optimization 1: Element-length (Offset_from_tag) When neither prefix nor suffix length is known but the element itself has a fixed codepoint length, only 1 tag should be needed instead of 2. Current: ensure_mem 2 (1 tag, end = tag + 1; every tag costs its canonical cell plus a working register, copied on accept). Goal: ensure_mem 1 (an unambiguous tag written directly to its canonical cell). *) let%expect_test "optim: element-length (Offset_from_tag)" = (match%sedlex_test buf with | Plus 'a', ('b' as x), Plus 'c' -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t1}"]; state2 [label="2"]; state2 -> state3 [label="'c'"]; state3 [label="3\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state3 -> state3 [label="'c'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = (Sedlexing.__private__mem_pos buf 0) + 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Optimization 2: Or-pattern offset propagation When an or-pattern is at the top level of a rule, as-bindings covering the whole branch should get Start_plus 0 / End_minus 0 without allocating any tags. Current: 0 tags (offsets already known at top level). Goal: already optimal. *) let%expect_test "optim: or-pattern offset propagation" = (match%sedlex_test buf with | (Plus 'a' as x) | (Plus 'b' as x) -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state2 [label="'b'"]; state1 [label="1\n[rule 0]", shape=doublecircle]; state1 -> state1 [label="'a'"]; state2 [label="2\n[rule 0]", shape=doublecircle]; state2 -> state2 [label="'b'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = Sedlexing.mark buf 0; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_2 buf = Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Optimization 3: Discriminator elision When both branches of an or-pattern produce identical position expressions, the discriminator tag should be skipped entirely. Current: 0 tags (both branches yield Start_plus 0, End_minus 0). Goal: already optimal. *) let%expect_test "optim: discriminator elision" = (match%sedlex_test buf with | (Plus '0' .. '9' as x) | (Plus 'a' .. 'z' as x) -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'0'-'9'"]; state0 -> state2 [label="'a'-'z'"]; state1 [label="1\n[rule 0]", shape=doublecircle]; state1 -> state1 [label="'0'-'9'"]; state2 [label="2\n[rule 0]", shape=doublecircle]; state2 -> state2 [label="'a'-'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = Sedlexing.mark buf 0; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_2 buf = Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Optimization 4: Intra-rule tag coalescing Tags with identical occurrence signatures should share one memory cell. Here x_end and y_start fire on the same transitions. Current: ensure_mem 2 (1 tag: x_start=0, x_end=y_start via Tag offset, y_end=lexeme_length; canonical cell plus working register). Goal: ensure_mem 1 (the tag written directly to its canonical cell). *) let%expect_test "optim: intra-rule tag coalescing" = (match%sedlex_test buf with | (Plus 'a' as x), (Plus 'b' as y) -> ignore (x, y) | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t1}"]; state2 [label="2\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state2 -> state2 [label="'b'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let y = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (x, y) | _ -> () |}] (* Optimization 5: Cross-rule cell sharing (graph coloring) Non-interfering rules should reuse the same memory cells. Rule 0 and rule 1 never co-exist in the same DFA state (beyond state 0), so their tags can share cells. Current: ensure_mem 8 (2 tags per rule, start + end of the variable-length binding, each with a canonical cell and a working register). Goal: ensure_mem 2 (tags written directly to their canonical cells, and cells shared across non-interfering rules). *) let%expect_test "optim: cross-rule cell sharing" = (match%sedlex_test buf with | Plus 'a', (Plus 'b' as x), Plus 'c' -> ignore x | Plus 'd', (Plus 'e' as y), Plus 'f' -> ignore y | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state4 [label="'d'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t4}"]; state2 [label="2"]; state2 -> state2 [label="'b'"]; state2 -> state3 [label="'c' {t5}"]; state3 [label="3\n[rule 0]\n{t1<-t5,t0<-t4}", shape=doublecircle]; state3 -> state3 [label="'c'"]; state4 [label="4"]; state4 -> state4 [label="'d'"]; state4 -> state5 [label="'e' {t6}"]; state5 [label="5"]; state5 -> state5 [label="'e'"]; state5 -> state6 [label="'f' {t7}"]; state6 [label="6\n[rule 1]\n{t3<-t7,t2<-t6}", shape=doublecircle]; state6 -> state6 [label="'f'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 4; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 5; __sedlex_state_3 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = Sedlexing.__private__copy_mem buf 1 5; Sedlexing.__private__copy_mem buf 0 4; Sedlexing.mark buf 0; (match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_4 buf = match __sedlex_partition_5 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 6; __sedlex_state_5 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_5 buf = match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_5 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 7; __sedlex_state_6 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_6 buf = Sedlexing.__private__copy_mem buf 3 7; Sedlexing.__private__copy_mem buf 2 6; Sedlexing.mark buf 1; (match __sedlex_partition_7 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_6 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 8; __sedlex_state_0 buf with | 0 -> let x = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.__private__mem_pos buf 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | 1 -> let y = let __s = Sedlexing.__private__mem_pos buf 2 in let __e = Sedlexing.__private__mem_pos buf 3 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore y | _ -> () |}] (* Optimization 6: Dead tag elimination Tags on transitions where the tag's owning rule can no longer reach a final state should be removed. Rule 0 has a binding on Plus 'b'; rule 1 does not. Both share the Plus 'a', Plus 'b' prefix in the DFA. Current: ensure_mem 4. The start tag is set once, on the transition entering the 'b's, shared by both rules; the end tag is set on the 'c' transition only rule 0 takes, and nothing is written on the 'd' transition only rule 1 takes. Goal: reached here; in general it needs a liveness analysis. *) let%expect_test "optim: dead tag elimination" = (match%sedlex_test buf with | Plus 'a', (Plus 'b' as x), Plus 'c' -> ignore x | Plus 'a', Plus 'b', Plus 'd' -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t2}"]; state2 [label="2"]; state2 -> state2 [label="'b'"]; state2 -> state3 [label="'c' {t3}"]; state2 -> state4 [label="'d'"]; state3 [label="3\n[rule 0]\n{t1<-t3,t0<-t2}", shape=doublecircle]; state3 -> state3 [label="'c'"]; state4 [label="4\n[rule 1]", shape=doublecircle]; state4 -> state4 [label="'d'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 2; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 3; __sedlex_state_3 buf) | 2 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = Sedlexing.__private__copy_mem buf 1 3; Sedlexing.__private__copy_mem buf 0 2; Sedlexing.mark buf 0; (match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_4 buf = Sedlexing.mark buf 1; (match __sedlex_partition_5 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 4; __sedlex_state_0 buf with | 0 -> let x = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.__private__mem_pos buf 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | 1 -> () | _ -> () |}] (* End of input never shares a transition with characters: [eof | 'b'] is one character set in the NFA and two transitions in the DFA. *) let%expect_test "end of input gets its own transition" = (match%sedlex_test buf with | (Plus 'a' as x), (eof | 'b') -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="EOF {t1}"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t1}"]; state2 [label="2\n[rule 0]\n{t0<-t1}", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_pos buf 1; Sedlexing.__private__copy_mem buf 0 1; 0) | 1 -> __sedlex_state_1 buf | 2 -> (Sedlexing.__private__set_mem_prev_pos buf 1; Sedlexing.__private__copy_mem buf 0 1; 0) | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Optimization 7: Self-loop tag delay (Set_prev) Tags on a self-loop that also appear on all entering transitions should be delayed to exit transitions as Set_prev. Current: ensure_mem 2; no write on the self-loop, the tag is set once on the transition leaving it (set_mem_prev_pos). Goal: reached (O(1) instead of O(n) writes). *) let%expect_test "optim: self-loop tag delay" = (match%sedlex_test buf with (Plus 'a' as x), Plus 'b' -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t1}"]; state2 [label="2\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state2 -> state2 [label="'b'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Optimization 8: Static offsets around a variable-length capture Fixed-length neighbours give every binding a static offset, even the variable-length one in the middle. Current: 0 tags (all offsets known: x=0..1, y=1..end-1, z=end-1..end). Goal: already optimal. *) let%expect_test "optim: static offsets around a variable-length capture" = (match%sedlex_test buf with | ('a' as x), (Plus 'b' as y), ('c' as z) -> ignore (x, y, z) | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2"]; state2 -> state2 [label="'b'"]; state2 -> state3 [label="'c'"]; state3 [label="3\n[rule 0]", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | 1 -> 0 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let y = let __s = 1 in let __e = (Sedlexing.lexeme_length buf) - 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let z = let __s = (Sedlexing.lexeme_length buf) - 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (x, y, z) | _ -> () |}] (* Optimization 9: Delayed tag with backtracking Opt at the end means the DFA can accept at two states (with or without the optional 'a'). The tag delayed out of the 'a' loop is materialized into its canonical cell at each accept, so backtracking to the earlier one sees the right value (the runtime check is in test/basic.ml). Current: ensure_mem 2 (1 tag, canonical cell plus working register; x: start=0, end=tag0; y: start=tag0, end=lexeme_length). *) let%expect_test "optim: delayed tag with backtracking" = (match%sedlex_test buf with | (Plus 'a' as x), ((Plus 'b', Opt 'a') as y) -> ignore (x, y) | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state1 [label="1"]; state1 -> state1 [label="'a'"]; state1 -> state2 [label="'b' {t1}"]; state2 [label="2\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state2 -> state3 [label="'a'"]; state2 -> state2 [label="'b'"]; state3 [label="3\n[rule 0]\n{t0<-t1}", shape=doublecircle]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_2 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__copy_mem buf 0 1; 0) | 1 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let y = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (x, y) | _ -> () |}] let%expect_test "Rep fixed-length prefix enables Start_plus" = (* Rep('0'..'9', 3..3) has fixed length 3. Current: 0 tags (prefix=3, suffix=0 both known → Start_plus/End_minus). Goal: already optimal. *) (match%sedlex_test buf with | Rep ('0' .. '9', 3 .. 3), (Plus 'a' .. 'z' as x) -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'0'-'9'"]; state1 [label="1"]; state1 -> state2 [label="'0'-'9'"]; state2 [label="2"]; state2 -> state3 [label="'0'-'9'"]; state3 [label="3"]; state3 -> state4 [label="'a'-'z'"]; state4 [label="4\n[rule 0]", shape=doublecircle]; state4 -> state4 [label="'a'-'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = Sedlexing.mark buf 0; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> let x = let __s = 3 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] (* Counterpart of "Rep fixed-length prefix enables Start_plus": a Rep with a variable range (2..3) has no fixed length, so the capture's start needs a tag. *) let%expect_test "Rep variable-length prefix forces a tag" = (match%sedlex_test buf with | Rep ('0' .. '9', 2 .. 3), (Plus 'a' .. 'z' as x) -> ignore x | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'0'-'9'"]; state1 [label="1"]; state1 -> state2 [label="'0'-'9'"]; state2 [label="2"]; state2 -> state3 [label="'0'-'9'"]; state2 -> state4 [label="'a'-'z' {t1}"]; state3 [label="3"]; state3 -> state4 [label="'a'-'z' {t1}"]; state4 [label="4\n[rule 0]\n{t0<-t1}", shape=doublecircle]; state4 -> state4 [label="'a'-'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | 1 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_4 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_prev_pos buf 1; __sedlex_state_4 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = Sedlexing.__private__copy_mem buf 0 1; Sedlexing.mark buf 0; (match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 2; __sedlex_state_0 buf with | 0 -> let x = let __s = Sedlexing.__private__mem_pos buf 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore x | _ -> () |}] let%expect_test "as binding: or-chain then nested or on right" = (match%sedlex_test buf with | ("ab" as x), ("ef" as y) | ("a" as x), ("bef" as y) | (("cde" as x), "f" | ("c" as x), "ef"), ("gh" as y) -> ignore (x, y) | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'a'"]; state0 -> state5 [label="'c'"]; state1 [label="1"]; state1 -> state2 [label="'b'"]; state2 [label="2"]; state2 -> state3 [label="'e'"]; state3 [label="3"]; state3 -> state4 [label="'f'"]; state4 [label="4\n[rule 0]\n{d1=0}", shape=doublecircle]; state5 [label="5"]; state5 -> state6 [label="'d'"]; state5 -> state11 [label="'e'"]; state6 [label="6"]; state6 -> state7 [label="'e'"]; state7 [label="7"]; state7 -> state8 [label="'f'"]; state8 [label="8"]; state8 -> state9 [label="'g' {d2=0}"]; state9 [label="9"]; state9 -> state10 [label="'h'"]; state10 [label="10\n[rule 0]\n{d1=2,t0<-t2}", shape=doublecircle]; state11 [label="11"]; state11 -> state12 [label="'f'"]; state12 [label="12"]; state12 -> state9 [label="'g' {d2=1}"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_5 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_value buf 1 0; 0) | _ -> Sedlexing.backtrack buf and __sedlex_state_5 buf = match __sedlex_partition_5 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_6 buf | 1 -> __sedlex_state_11 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_6 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_7 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_7 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_8 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_8 buf = match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_value buf 2 0; __sedlex_state_9 buf) | _ -> Sedlexing.backtrack buf and __sedlex_state_9 buf = match __sedlex_partition_7 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_value buf 1 2; Sedlexing.__private__copy_mem buf 0 2; 0) | _ -> Sedlexing.backtrack buf and __sedlex_state_11 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_12 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_12 buf = match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_value buf 2 1; __sedlex_state_9 buf) | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 3; __sedlex_state_0 buf with | 0 -> let x = if (Sedlexing.__private__mem_value buf 1) = 0 then let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 2 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } else if (Sedlexing.__private__mem_value buf 1) = 1 then (let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 3 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) else if ((Sedlexing.__private__mem_value buf 1) = 2) && ((Sedlexing.__private__mem_value buf 0) = 0) then (let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 3 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) else (let __s = 0 in let __e = (Sedlexing.lexeme_length buf) - 4 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) in let y = if (Sedlexing.__private__mem_value buf 1) = 0 then let __s = 2 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } else if (Sedlexing.__private__mem_value buf 1) = 1 then (let __s = 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) else (let __s = (Sedlexing.lexeme_length buf) - 2 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) }) in ignore (x, y) | _ -> () |}] (* ------------------------------------------------------------------------ *) (* Edge cases pinned at the generated-code level. *) (* Nullable-only rule. When every rule matches the empty string, state 0 is an accepting sink: no state function is generated for it, and the block returns the sink's rule index directly, with no `let rec`. *) let%expect_test "nullable-only rule" = (match%sedlex_test buf with "" -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0\n[rule 0]", shape=doublecircle]; } CODE: match Sedlexing.start buf; 0 with | 0 -> () | _ -> () |}] (* End of input is read once: what an eof transition leads to has no transitions, so the DFA of [Plus eof] is the DFA of [eof]. It used to have an EOF self-edge, and the generated function called itself forever. *) let%expect_test "eof self-loop" = (match%sedlex_test buf with Plus eof -> () | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="EOF"]; state1 [label="1\n[rule 0]", shape=doublecircle]; } CODE: let __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> 0 | _ -> Sedlexing.backtrack buf in match Sedlexing.start buf; __sedlex_state_0 buf with | 0 -> () | _ -> () |}] sedlex-3.8.1/test/codegen/test_realistic.ml000066400000000000000000000202151526113672700207540ustar00rootroot00000000000000(* Realistic multi-rule lexer exercising many patterns simultaneously. Current: ensure_mem 4 (2 tags, each with its canonical cell and a working register). - Rule 0: (Plus 'a'..'z' as ns), '.', (Plus 'a'..'z' as name) → 1 tag (ns: start=0, end=tag0; name: start=tag0+1 via Tag offset, end=lexeme_length) - Rule 1: (Plus 'A'..'Z' as label), '=', (Plus '0'..'9' as value) → 1 tag (label: start=0, end=tag1; value: start=tag1+1 via Tag offset, end=lexeme_length) - Rule 2: "0x", (Plus hex as hex), ';' → 0 tags (prefix=2, suffix=1) - Rule 3: '(', ('a'..'z' as x), ',', ('a'..'z' as y), ')' → 0 tags (all fixed offsets) - Rule 4: (Plus digits as tok) | (Plus letters as tok) → 0 tags (discriminator elided) Each tag is set once, on the '.' or '=' transition leaving the loop. Remaining optimization goals: - Unambiguous tags written directly to their canonical cell (ensure_mem 4 → 2) - Cross-rule cell sharing → rules 0,1 share cells (ensure_mem 2 → 1) *) let%expect_test "realistic: multi-token lexer" = (match%sedlex_test buf with | (Plus 'a' .. 'z' as ns), '.', (Plus 'a' .. 'z' as name) -> ignore (ns, name) | (Plus 'A' .. 'Z' as label), '=', (Plus '0' .. '9' as value) -> ignore (label, value) | "0x", (Plus ('0' .. '9' | 'a' .. 'f') as hex), ';' -> ignore hex | '(', ('a' .. 'z' as x), ',', ('a' .. 'z' as y), ')' -> ignore (x, y) | (Plus '0' .. '9' as tok) | (Plus 'a' .. 'z' as tok) -> ignore tok | _ -> ()); [%expect {| DOT: digraph { rankdir=LR; node [shape=circle]; _start [shape=point]; _start -> state0; state0 [label="0"]; state0 -> state1 [label="'('"]; state0 -> state6 [label="'0'"]; state0 -> state7 [label="'1'-'9'"]; state0 -> state11 [label="'A'-'Z'"]; state0 -> state14 [label="'a'-'z'"]; state1 [label="1"]; state1 -> state2 [label="'a'-'z'"]; state2 [label="2"]; state2 -> state3 [label="','"]; state3 [label="3"]; state3 -> state4 [label="'a'-'z'"]; state4 [label="4"]; state4 -> state5 [label="')'"]; state5 [label="5\n[rule 3]", shape=doublecircle]; state6 [label="6\n[rule 4]", shape=doublecircle]; state6 -> state7 [label="'0'-'9'"]; state6 -> state8 [label="'x'"]; state7 [label="7\n[rule 4]", shape=doublecircle]; state7 -> state7 [label="'0'-'9'"]; state8 [label="8"]; state8 -> state9 [label="'0'-'9', 'a'-'f'"]; state9 [label="9"]; state9 -> state9 [label="'0'-'9', 'a'-'f'"]; state9 -> state10 [label="';'"]; state10 [label="10\n[rule 2]", shape=doublecircle]; state11 [label="11"]; state11 -> state12 [label="'=' {t2}"]; state11 -> state11 [label="'A'-'Z'"]; state12 [label="12"]; state12 -> state13 [label="'0'-'9'"]; state13 [label="13\n[rule 1]\n{t1<-t2}", shape=doublecircle]; state13 -> state13 [label="'0'-'9'"]; state14 [label="14\n[rule 4]", shape=doublecircle]; state14 -> state15 [label="'.' {t3}"]; state14 -> state14 [label="'a'-'z'"]; state15 [label="15"]; state15 -> state16 [label="'a'-'z'"]; state16 [label="16\n[rule 0]\n{t0<-t3}", shape=doublecircle]; state16 -> state16 [label="'a'-'z'"]; } CODE: let rec __sedlex_state_0 buf = match __sedlex_partition_1 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_1 buf | 1 -> __sedlex_state_6 buf | 2 -> __sedlex_state_7 buf | 3 -> __sedlex_state_11 buf | 4 -> __sedlex_state_14 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_1 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_2 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_2 buf = match __sedlex_partition_3 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_3 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_3 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_4 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_4 buf = match __sedlex_partition_4 (Sedlexing.__private__next_int buf) with | 0 -> 3 | _ -> Sedlexing.backtrack buf and __sedlex_state_6 buf = Sedlexing.mark buf 4; (match __sedlex_partition_5 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_7 buf | 1 -> __sedlex_state_8 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_7 buf = Sedlexing.mark buf 4; (match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_7 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_8 buf = match __sedlex_partition_7 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_9 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_9 buf = match __sedlex_partition_8 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_9 buf | 1 -> 2 | _ -> Sedlexing.backtrack buf and __sedlex_state_11 buf = match __sedlex_partition_9 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_prev_pos buf 2; __sedlex_state_12 buf) | 1 -> __sedlex_state_11 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_12 buf = match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_13 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_13 buf = Sedlexing.__private__copy_mem buf 1 2; Sedlexing.mark buf 1; (match __sedlex_partition_6 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_13 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_14 buf = Sedlexing.mark buf 4; (match __sedlex_partition_10 (Sedlexing.__private__next_int buf) with | 0 -> (Sedlexing.__private__set_mem_prev_pos buf 3; __sedlex_state_15 buf) | 1 -> __sedlex_state_14 buf | _ -> Sedlexing.backtrack buf) and __sedlex_state_15 buf = match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_16 buf | _ -> Sedlexing.backtrack buf and __sedlex_state_16 buf = Sedlexing.__private__copy_mem buf 0 3; Sedlexing.mark buf 0; (match __sedlex_partition_2 (Sedlexing.__private__next_int buf) with | 0 -> __sedlex_state_16 buf | _ -> Sedlexing.backtrack buf) in match Sedlexing.start buf; Sedlexing.__private__ensure_mem buf 4; __sedlex_state_0 buf with | 0 -> let ns = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 0 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let name = let __s = (Sedlexing.__private__mem_pos buf 0) + 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (ns, name) | 1 -> let label = let __s = 0 in let __e = Sedlexing.__private__mem_pos buf 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let value = let __s = (Sedlexing.__private__mem_pos buf 1) + 1 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (label, value) | 2 -> let hex = let __s = 2 in let __e = (Sedlexing.lexeme_length buf) - 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore hex | 3 -> let x = let __s = 1 in let __e = (Sedlexing.lexeme_length buf) - 3 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in let y = let __s = 3 in let __e = (Sedlexing.lexeme_length buf) - 1 in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore (x, y) | 4 -> let tok = let __s = 0 in let __e = Sedlexing.lexeme_length buf in { Sedlexing.lexbuf = buf; pos = __s; len = (__e - __s) } in ignore tok | _ -> () |}] sedlex-3.8.1/test/dune000066400000000000000000000002611526113672700146550ustar00rootroot00000000000000(library (name sedlex_test) (libraries sedlex) (inline_tests (deps UTF-8-test.txt)) (enabled_if (>= %{ocaml_version} 4.14)) (preprocess (pps sedlex.ppx ppx_expect))) sedlex-3.8.1/test/nested.ml000066400000000000000000000003551526113672700156170ustar00rootroot00000000000000let%expect_test _ = let lb = Sedlexing.Utf8.from_string "ab" in let res = match%sedlex lb with | 'a' -> ( match%sedlex lb with 'b' -> "ok" | _ -> "error") | _ -> "error" in print_endline res; [%expect {| ok |}] sedlex-3.8.1/test/ppx_test/000077500000000000000000000000001526113672700156465ustar00rootroot00000000000000sedlex-3.8.1/test/ppx_test/dune000066400000000000000000000002201526113672700165160ustar00rootroot00000000000000(library (name ppx_sedlex_test) (kind ppx_rewriter) (libraries ppxlib sedlex_ppx sedlex.compiler str) (preprocess (pps ppxlib.metaquot))) sedlex-3.8.1/test/ppx_test/ppx_sedlex_test.ml000066400000000000000000000035571526113672700214240ustar00rootroot00000000000000open Ppxlib module P = Sedlex_ppx.Ppx_sedlex module S = Sedlex_compiler.Sedlex let expand ~ctxt:_ expr = P.reset_state (); S.reset_tags (); let loc = Location.none in let code_expr, auto = P.handle_sedlex_match expr in let code_str = Pprintast.string_of_expression code_expr in let dot_str = S.dfa_to_dot auto in P.reset_state (); [%expr print_string "DOT:\n"; print_string [%e Ast_builder.Default.estring ~loc dot_str]; print_string "CODE:\n"; print_string [%e Ast_builder.Default.estring ~loc code_str]; print_newline ()] let strip_line_numbers s = let s = Str.global_replace (Str.regexp "line [0-9]+, ") "" s in let blank_digits m = let t = Str.matched_string m in String.init (String.length t) (fun i -> if t.[i] = '|' then '|' else ' ') in Str.global_substitute (Str.regexp "^ *[0-9]+ |") blank_digits s let expand_error ~ctxt:_ expr = P.reset_state (); S.reset_tags (); let loc = Location.none in let msg = try let _ = P.map_expression expr in "NO ERROR" with exn -> ( match Location.Error.of_exn exn with | Some _ -> let buf = Buffer.create 256 in let fmt = Format.formatter_of_buffer buf in Location.report_exception fmt exn; Format.pp_print_flush fmt (); strip_line_numbers (Buffer.contents buf) | None -> "Uncaught exception: " ^ Printexc.to_string exn) in P.reset_state (); [%expr print_string [%e Ast_builder.Default.estring ~loc msg]; print_newline ()] let ext = Extension.V3.declare "sedlex_test" Extension.Context.expression Ast_pattern.(single_expr_payload __) expand let ext_error = Extension.V3.declare "compile_error" Extension.Context.expression Ast_pattern.(single_expr_payload __) expand_error let () = Driver.register_transformation "sedlex_test" ~extensions:[ext; ext_error] sedlex-3.8.1/test/stress_test.ml000066400000000000000000000617361526113672700167310ustar00rootroot00000000000000let b = Buffer.create 4 let width = function | '\000' .. '\127' -> 1 | '\192' .. '\223' -> 2 | '\224' .. '\239' -> 3 | '\240' .. '\247' -> 4 | _ -> 1 let rec token buf = match%sedlex buf with | eof -> () | any -> Buffer.clear b; Buffer.add_utf_8_uchar b (Sedlexing.lexeme_char buf 0); Printf.printf "%s" (Buffer.contents b); token buf | _ -> assert false let%expect_test "stress_test" = let ic = open_in_bin "UTF-8-test.txt" in let content = really_input_string ic (in_channel_length ic) in close_in ic; let pos = ref 0 in let get_next () = if !pos = String.length content then None else ( let len = width content.[!pos] in let c = String.sub content !pos len in pos := !pos + len; Some c) in (get_next |> Seq.of_dispenser |> Seq.iter @@ fun c -> let lexbuf = Sedlexing.Utf8.from_string c in try token lexbuf with Sedlexing.MalFormed -> Printf.printf "[MF]"); [%expect {| UTF-8 decoder capability and stress test ---------------------------------------- Markus Kuhn - 2015-08-28 - CC BY 4.0 This test file can help you examine, how your UTF-8 decoder handles various types of correct, malformed, or otherwise interesting UTF-8 sequences. This file is not meant to be a conformance test. It does not prescribe any particular outcome. Therefore, there is no way to "pass" or "fail" this test file, even though the text does suggest a preferable decoder behaviour at some places. Its aim is, instead, to help you think about, and test, the behaviour of your UTF-8 decoder on a systematic collection of unusual inputs. Experience so far suggests that most first-time authors of UTF-8 decoders find at least one serious problem in their decoder using this file. The test lines below cover boundary conditions, malformed UTF-8 sequences, as well as correctly encoded UTF-8 sequences of Unicode code points that should never occur in a correct UTF-8 file. According to ISO 10646-1:2000, sections D.7 and 2.3c, a device receiving UTF-8 shall interpret a "malformed sequence in the same way that it interprets a character that is outside the adopted subset" and "characters that are not within the adopted subset shall be indicated to the user" by a receiving device. One commonly used approach in UTF-8 decoders is to replace any malformed UTF-8 sequence by a replacement character (U+FFFD), which looks a bit like an inverted question mark, or a similar symbol. It might be a good idea to visually distinguish a malformed UTF-8 sequence from a correctly encoded Unicode character that is just not available in the current font but otherwise fully legal, even though ISO 10646-1 doesn't mandate this. In any case, just ignoring malformed sequences or unavailable characters does not conform to ISO 10646, will make debugging more difficult, and can lead to user confusion. Please check, whether a malformed UTF-8 sequence is (1) represented at all, (2) represented by exactly one single replacement character (or equivalent signal), and (3) the following quotation mark after an illegal UTF-8 sequence is correctly displayed, i.e. proper resynchronization takes place immediately after any malformed sequence. This file says "THE END" in the last line, so if you don't see that, your decoder crashed somehow before, which should always be cause for concern. All lines in this file are exactly 79 characters long (plus the line feed). In addition, all lines end with "|", except for the two test lines 2.1.1 and 2.2.1, which contain non-printable ASCII controls U+0000 and U+007F. If you display this file with a fixed-width font, these "|" characters should all line up in column 79 (right margin). This allows you to test quickly, whether your UTF-8 decoder finds the correct number of characters in every line, that is whether each malformed sequences is replaced by a single replacement character. Note that, as an alternative to the notion of malformed sequence used here, it is also a perfectly acceptable (and in some situations even preferable) solution to represent each individual byte of a malformed sequence with a replacement character. If you follow this strategy in your decoder, then please ignore the "|" column. Here come the tests: | | 1 Some correct UTF-8 text | | You should see the Greek word 'kosme': "κόσμε" | | 2 Boundary condition test cases | | 2.1 First possible sequence of a certain length | | 2.1.1 1 byte (U-00000000): "" 2.1.2 2 bytes (U-00000080): "€" | 2.1.3 3 bytes (U-00000800): "à €" | 2.1.4 4 bytes (U-00010000): "ð�€€" | 2.1.5 5 bytes (U-00200000): "[MF][MF][MF][MF][MF]" | 2.1.6 6 bytes (U-04000000): "[MF][MF][MF][MF][MF][MF]" | | 2.2 Last possible sequence of a certain length | | 2.2.1 1 byte (U-0000007F): "" 2.2.2 2 bytes (U-000007FF): "ß¿" | 2.2.3 3 bytes (U-0000FFFF): "ï¿¿" | 2.2.4 4 bytes (U-001FFFFF): "[MF]" | 2.2.5 5 bytes (U-03FFFFFF): "[MF][MF][MF][MF][MF]" | 2.2.6 6 bytes (U-7FFFFFFF): "[MF][MF][MF][MF][MF][MF]" | | 2.3 Other boundary conditions | | 2.3.1 U-0000D7FF = ed 9f bf = "퟿" | 2.3.2 U-0000E000 = ee 80 80 = "" | 2.3.3 U-0000FFFD = ef bf bd = "�" | 2.3.4 U-0010FFFF = f4 8f bf bf = "ô�¿¿" | 2.3.5 U-00110000 = f4 90 80 80 = "[MF]" | | 3 Malformed sequences | | 3.1 Unexpected continuation bytes | | Each unexpected continuation byte should be separately signalled as a | malformed sequence of its own. | | 3.1.1 First continuation byte 0x80: "[MF]" | 3.1.2 Last continuation byte 0xbf: "[MF]" | | 3.1.3 2 continuation bytes: "[MF][MF]" | 3.1.4 3 continuation bytes: "[MF][MF][MF]" | 3.1.5 4 continuation bytes: "[MF][MF][MF][MF]" | 3.1.6 5 continuation bytes: "[MF][MF][MF][MF][MF]" | 3.1.7 6 continuation bytes: "[MF][MF][MF][MF][MF][MF]" | 3.1.8 7 continuation bytes: "[MF][MF][MF][MF][MF][MF][MF]" | | 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf): | | "[MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF] | [MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF] | [MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF] | [MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF]" | | 3.2 Lonely start characters | | 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf), | each followed by a space character: | | "[MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF] | [MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF]" | | 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef), | each followed by a space character: | | "[MF] [MF] [MF] [MF] [MF] [MF] [MF] [MF] " | | 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7), | each followed by a space character: | | "[MF][MF][MF][MF]" | | 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb), | each followed by a space character: | | "[MF] [MF] [MF] [MF] " | | 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd), | each followed by a space character: | | "[MF] [MF] " | | 3.3 Sequences with last continuation byte missing | | All bytes of an incomplete sequence should be signalled as a single | malformed sequence, i.e., you should see only a single replacement | character in each of the next 10 tests. (Characters as in section 2) | | 3.3.1 2-byte sequence with last byte missing (U+0000): "[MF] | 3.3.2 3-byte sequence with last byte missing (U+0000): "[MF] | 3.3.3 4-byte sequence with last byte missing (U+0000): "[MF] | 3.3.4 5-byte sequence with last byte missing (U+0000): "[MF][MF][MF][MF]" | 3.3.5 6-byte sequence with last byte missing (U+0000): "[MF][MF][MF][MF][MF]" | 3.3.6 2-byte sequence with last byte missing (U-000007FF): "[MF] | 3.3.7 3-byte sequence with last byte missing (U-0000FFFF): "[MF] | 3.3.8 4-byte sequence with last byte missing (U-001FFFFF): "[MF] | 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): "[MF][MF][MF][MF]" | 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "[MF][MF][MF][MF][MF]" | | 3.4 Concatenation of incomplete sequences | | All the 10 sequences of 3.3 concatenated, you should see 10 malformed | sequences being signalled: | | "[MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF][MF]" | | 3.5 Impossible bytes | | The following two bytes cannot appear in a correct UTF-8 string | | 3.5.1 fe = "[MF]" | 3.5.2 ff = "[MF]" | 3.5.3 fe fe ff ff = "[MF][MF][MF][MF]" | | 4 Overlong sequences | | The following sequences are not malformed according to the letter of | the Unicode 2.0 standard. However, they are longer then necessary and | a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 | decoder" should reject them just like malformed sequences for two | reasons: (1) It helps to debug applications if overlong sequences are | not treated as valid representations of characters, because this helps | to spot problems more quickly. (2) Overlong sequences provide | alternative representations of characters, that could maliciously be | used to bypass filters that check only for ASCII characters. For | instance, a 2-byte encoded line feed (LF) would not be caught by a | line counter that counts only 0x0a bytes, but it would still be | processed as a line feed by an unsafe UTF-8 decoder later in the | pipeline. From a security point of view, ASCII compatibility of UTF-8 | sequences means also, that ASCII characters are *only* allowed to be | represented by ASCII bytes in the range 0x00-0x7f. To ensure this | aspect of ASCII compatibility, use only "safe UTF-8 decoders" that | reject overlong UTF-8 sequences for which a shorter encoding exists. | | 4.1 Examples of an overlong ASCII character | | With a safe UTF-8 decoder, all of the following five overlong | representations of the ASCII character slash ("/") should be rejected | like a malformed UTF-8 sequence, for instance by substituting it with | a replacement character. If you see a slash below, you do not have a | safe UTF-8 decoder! | | 4.1.1 U+002F = c0 af = "[MF]" | 4.1.2 U+002F = e0 80 af = "[MF]" | 4.1.3 U+002F = f0 80 80 af = "[MF]" | 4.1.4 U+002F = f8 80 80 80 af = "[MF][MF][MF][MF][MF]" | 4.1.5 U+002F = fc 80 80 80 80 af = "[MF][MF][MF][MF][MF][MF]" | | 4.2 Maximum overlong sequences | | Below you see the highest Unicode value that is still resulting in an | overlong sequence if represented with the given number of bytes. This | is a boundary test for safe UTF-8 decoders. All five characters should | be rejected like malformed UTF-8 sequences. | | 4.2.1 U-0000007F = c1 bf = "[MF]" | 4.2.2 U-000007FF = e0 9f bf = "[MF]" | 4.2.3 U-0000FFFF = f0 8f bf bf = "[MF]" | 4.2.4 U-001FFFFF = f8 87 bf bf bf = "[MF][MF][MF][MF][MF]" | 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf = "[MF][MF][MF][MF][MF][MF]" | | 4.3 Overlong representation of the NUL character | | The following five sequences should also be rejected like malformed | UTF-8 sequences and should not be treated like the ASCII NUL | character. | | 4.3.1 U+0000 = c0 80 = "[MF]" | 4.3.2 U+0000 = e0 80 80 = "[MF]" | 4.3.3 U+0000 = f0 80 80 80 = "[MF]" | 4.3.4 U+0000 = f8 80 80 80 80 = "[MF][MF][MF][MF][MF]" | 4.3.5 U+0000 = fc 80 80 80 80 80 = "[MF][MF][MF][MF][MF][MF]" | | 5 Illegal code positions | | The following UTF-8 sequences should be rejected like malformed | sequences, because they never represent valid ISO 10646 characters and | a UTF-8 decoder that accepts them might introduce security problems | comparable to overlong UTF-8 sequences. | | 5.1 Single UTF-16 surrogates | | 5.1.1 U+D800 = ed a0 80 = "[MF]" | 5.1.2 U+DB7F = ed ad bf = "[MF]" | 5.1.3 U+DB80 = ed ae 80 = "[MF]" | 5.1.4 U+DBFF = ed af bf = "[MF]" | 5.1.5 U+DC00 = ed b0 80 = "[MF]" | 5.1.6 U+DF80 = ed be 80 = "[MF]" | 5.1.7 U+DFFF = ed bf bf = "[MF]" | | 5.2 Paired UTF-16 surrogates | | 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = "[MF][MF]" | 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = "[MF][MF]" | 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = "[MF][MF]" | 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = "[MF][MF]" | 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = "[MF][MF]" | 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = "[MF][MF]" | 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = "[MF][MF]" | 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = "[MF][MF]" | | 5.3 Noncharacter code positions | | The following "noncharacters" are "reserved for internal use" by | applications, and according to older versions of the Unicode Standard | "should never be interchanged". Unicode Corrigendum #9 dropped the | latter restriction. Nevertheless, their presence in incoming UTF-8 data | can remain a potential security risk, depending on what use is made of | these codes subsequently. Examples of such internal use: | | - Some file APIs with 16-bit characters may use the integer value -1 | = U+FFFF to signal an end-of-file (EOF) or error condition. | | - In some UTF-16 receivers, code point U+FFFE might trigger a | byte-swap operation (to convert between UTF-16LE and UTF-16BE). | | With such internal use of noncharacters, it may be desirable and safer | to block those code points in UTF-8 decoders, as they should never | occur legitimately in incoming UTF-8 data, and could trigger unsafe | behaviour in subsequent processing. | | Particularly problematic noncharacters in 16-bit applications: | | 5.3.1 U+FFFE = ef bf be = "￾" | 5.3.2 U+FFFF = ef bf bf = "ï¿¿" | | Other noncharacters: | | 5.3.3 U+FDD0 .. U+FDEF = "ï·�﷑﷒﷓﷔﷕﷖﷗﷘﷙﷚﷛﷜ï·�﷞﷟﷠﷡﷢﷣﷤﷥﷦﷧﷨﷩﷪﷫﷬﷭﷮﷯"| | 5.3.4 U+nFFFE U+nFFFF (for n = 1..10) | | "🿾🿿𯿾𯿿𿿾𿿿ñ�¿¾ñ�¿¿ñŸ¿¾ñŸ¿¿ñ¯¿¾ñ¯¿¿ñ¿¿¾ñ¿¿¿ò�¿¾ò�¿¿ | òŸ¿¾òŸ¿¿ò¯¿¾ò¯¿¿ò¿¿¾ò¿¿¿ó�¿¾ó�¿¿óŸ¿¾óŸ¿¿ó¯¿¾ó¯¿¿ó¿¿¾ó¿¿¿ô�¿¾ô�¿¿" | | THE END | |}] sedlex-3.8.1/test/utf8.ml000066400000000000000000000030161526113672700152200ustar00rootroot00000000000000open Printf let next_tok buf = let open Sedlexing.Utf8 in match%sedlex buf with | "a", Utf8 (Chars "+-×÷") -> sprintf "with Chars: %s" (lexeme buf) | "b", Utf8 ("+" | "-" | "×" | "÷") -> sprintf "with or_pattern: %s" (lexeme buf) | Latin1 "\xc0", Utf8 "À", Ascii (Utf8 (Latin1 (Utf8 (Chars "À")))) -> sprintf "mixed encoding: %s" (lexeme buf) | Ascii (Star '\x00' .. '\x7f') -> sprintf "only ascii: %s" (lexeme buf) | Utf8 (Star '\x00' .. '\x7f') -> assert false (* utf8 char interval can only match ascii because of the OCaml lexer. The regexp above should match instead *) | Latin1 (Star '\x00' .. '\xff') -> sprintf "only latin1: %s" (lexeme buf) | _ -> failwith (sprintf "Unexpected character: %s" (lexeme buf)) let%expect_test _ = Sedlexing.Utf8.from_string "a+" |> next_tok |> print_string; [%expect {| with Chars: a+ |}]; Sedlexing.Utf8.from_string "a÷" |> next_tok |> print_string; [%expect {| with Chars: a÷ |}]; Sedlexing.Utf8.from_string "b+" |> next_tok |> print_string; [%expect {| with or_pattern: b+ |}]; Sedlexing.Utf8.from_string "b÷" |> next_tok |> print_string; [%expect {| with or_pattern: b÷ |}]; Sedlexing.Utf8.from_string "ÀÀÀ" |> next_tok |> print_string; [%expect {| mixed encoding: ÀÀÀ |}]; Sedlexing.Utf8.from_string "az\x7f" |> next_tok |> String.escaped |> print_string; [%expect {| only ascii: az\127 |}]; Sedlexing.Utf8.from_string "az\u{c0}" |> next_tok |> print_string; [%expect {| only latin1: azÀ |}]