Skip to content

Update unicode #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
toots merged 1 commit into from
Aug 9, 2026
Merged

Update unicode #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
2.1.0 (unreleased)
------------------

- Update the Unicode Character Database from 3.2 to 17.0.0, and add a
`update-unicode` dune alias to refresh it (#3).
- Generate `UCharInfo.script_type` and `UCharInfo.version_type` from the
Unicode data instead of hardcoding them. Scripts added since Unicode 3.2 are
now recognized, `` `v1_0 `` is gone (the UCD tracks ages from 1.1 onwards)
and the version tags run up to `` `v17_0 ``.
- Fix canonical composition wrongly composing across a blocking character:
`NFC("େ̴ା")` no longer yields `ୋ̴` (UAX #15 D115).
- Fix charmap generation with dune 3.24, which expands `%{deps}` to a path
with a directory component (#13).
- Add embedded APIs to be able to build binaries with the shared data built-in
Expand Down
22 changes: 22 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,28 @@ See `camomile.ml` file.

See https://github.com/savonet/Camomile

### Updating the Unicode data

The Unicode Character Database is vendored under `src/unidata` and
`test/unidata` so that builds stay offline. To refresh it:

```
CAMOMILE_UPDATE_UNICODE=1 dune build @update-unicode --auto-promote
```

This downloads the latest published release and rewrites the vendored files in
place. Without `--auto-promote` it only reports which files are out of date.
`CAMOMILE_UPDATE_UNICODE` is what gates the download: with it unset, no build
ever reaches the network. There is no way to ask for a specific release — the
vendored files are the pin, so downgrading means reverting the commit.

`UCharInfo.script_type` and `UCharInfo.version_type` are generated from the
data, so new scripts and versions are picked up without touching any source.

The UCA conformance suite does not pass and is excluded from `dune runtest`;
run it with `dune build @uca-conformance`. See `test/tester/test_uCol.ml` for
what is broken.

## Author

Camomile is currently maintained by Romain Beauxis <romain.beauxis@gmail.com>
Expand Down
13 changes: 13 additions & 0 deletions dune
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
; `dune build` builds every declared target in the tree, so the download in
; src/unidata and test/unidata has to be gated on a variable -- otherwise no
; one could build Camomile offline. Say so rather than passing silently.

(rule
(alias update-unicode)
(enabled_if
(= %{env:CAMOMILE_UPDATE_UNICODE=} ""))
(action
(progn
(echo
"update-unicode: nothing checked. Re-run with CAMOMILE_UPDATE_UNICODE=1 to compare against unicode.org, and add --auto-promote to update in place.\n")
(run false))))
16 changes: 16 additions & 0 deletions src/dune
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,22 @@
(echo "let prefix = \"%{env:CAMOMILE_PREFIX=/usr}/share/camomile\"\n")
(cat configImpl.env.ml.in)))))

(rule
(target script_type.ml)
(deps tools/gen_unidata.exe unidata/Scripts.txt)
(action
(with-stdout-to
%{target}
(run tools/gen_unidata.exe --gen-script-type unidata/Scripts.txt))))

(rule
(target version_type.ml)
(deps tools/gen_unidata.exe unidata/DerivedAge.txt)
(action
(with-stdout-to
%{target}
(run tools/gen_unidata.exe --gen-version-type unidata/DerivedAge.txt))))

; -b 0 emits one string literal per file instead of 4096-byte chunks: read
; returns the literal itself and the data stays demand-paged from the
; executable, rather than being faulted in wholesale by the GC's root scan.
Expand Down
243 changes: 5 additions & 238 deletions src/internal/unidata.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,52 +74,7 @@ module type Type = sig
val num_of_cat : general_category_type -> int
val cat_of_num : int -> general_category_type

type script_type =
[ `Common
| `Inherited
| `Latin
| `Greek
| `Cyrillic
| `Armenian
| `Hebrew
| `Arabic
| `Syriac
| `Thaana
| `Devanagari
| `Bengali
| `Gurmukhi
| `Gujarati
| `Oriya
| `Tamil
| `Telugu
| `Kannada
| `Malayalam
| `Sinhala
| `Thai
| `Lao
| `Tibetan
| `Myanmar
| `Georgian
| `Hangul
| `Ethiopic
| `Cherokee
| `Canadian_Aboriginal
| `Ogham
| `Runic
| `Khmer
| `Mongolian
| `Hiragana
| `Katakana
| `Bopomofo
| `Han
| `Yi
| `Old_Italic
| `Gothic
| `Deseret
| `Tagalog
| `Hanunoo
| `Buhid
| `Tagbanwa ]
type script_type = Script_type.t

val script_of_name : string -> script_type
val script_of_num : int -> script_type
Expand Down Expand Up @@ -331,199 +286,11 @@ module Make (Config : Config.Type) = struct

let cat_of_num i = cat_of_num_tbl.(i)

type script_type =
[ `Common
| `Inherited
| `Latin
| `Greek
| `Cyrillic
| `Armenian
| `Hebrew
| `Arabic
| `Syriac
| `Thaana
| `Devanagari
| `Bengali
| `Gurmukhi
| `Gujarati
| `Oriya
| `Tamil
| `Telugu
| `Kannada
| `Malayalam
| `Sinhala
| `Thai
| `Lao
| `Tibetan
| `Myanmar
| `Georgian
| `Hangul
| `Ethiopic
| `Cherokee
| `Canadian_Aboriginal
| `Ogham
| `Runic
| `Khmer
| `Mongolian
| `Hiragana
| `Katakana
| `Bopomofo
| `Han
| `Yi
| `Old_Italic
| `Gothic
| `Deseret
| `Tagalog
| `Hanunoo
| `Buhid
| `Tagbanwa ]

let script_of_name name =
match String.lowercase_ascii name with
| "common" -> `Common
| "inherited" -> `Inherited
| "latin" -> `Latin
| "greek" -> `Greek
| "cyrillic" -> `Cyrillic
| "armenian" -> `Armenian
| "hebrew" -> `Hebrew
| "arabic" -> `Arabic
| "syriac" -> `Syriac
| "thaana" -> `Thaana
| "devanagari" -> `Devanagari
| "bengali" -> `Bengali
| "gurmukhi" -> `Gurmukhi
| "gujarati" -> `Gujarati
| "oriya" -> `Oriya
| "tamil" -> `Tamil
| "telugu" -> `Telugu
| "kannada" -> `Kannada
| "malayalam" -> `Malayalam
| "sinhala" -> `Sinhala
| "thai" -> `Thai
| "lao" -> `Lao
| "tibetan" -> `Tibetan
| "myanmar" -> `Myanmar
| "georgian" -> `Georgian
| "hangul" -> `Hangul
| "ethiopic" -> `Ethiopic
| "cherokee" -> `Cherokee
| "canadian_aboriginal" -> `Canadian_Aboriginal
| "ogham" -> `Ogham
| "runic" -> `Runic
| "khmer" -> `Khmer
| "mongolian" -> `Mongolian
| "hiragana" -> `Hiragana
| "katakana" -> `Katakana
| "bopomofo" -> `Bopomofo
| "han" -> `Han
| "yi" -> `Yi
| "old_italic" -> `Old_Italic
| "gothic" -> `Gothic
| "deseret" -> `Deseret
| "tagalog" -> `Tagalog
| "hanunoo" -> `Hanunoo
| "buhid" -> `Buhid
| "tagbanwa" -> `Tagbanwa
| _ -> raise Not_found

let num_of_script = function
| `Common -> 0
| `Inherited -> 1
| `Latin -> 2
| `Greek -> 3
| `Cyrillic -> 4
| `Armenian -> 5
| `Hebrew -> 6
| `Arabic -> 7
| `Syriac -> 8
| `Thaana -> 9
| `Devanagari -> 10
| `Bengali -> 11
| `Gurmukhi -> 12
| `Gujarati -> 13
| `Oriya -> 14
| `Tamil -> 15
| `Telugu -> 16
| `Kannada -> 17
| `Malayalam -> 18
| `Sinhala -> 19
| `Thai -> 20
| `Lao -> 21
| `Tibetan -> 22
| `Myanmar -> 23
| `Georgian -> 24
| `Hangul -> 25
| `Ethiopic -> 26
| `Cherokee -> 27
| `Canadian_Aboriginal -> 28
| `Ogham -> 29
| `Runic -> 30
| `Khmer -> 31
| `Mongolian -> 32
| `Hiragana -> 33
| `Katakana -> 34
| `Bopomofo -> 35
| `Han -> 36
| `Yi -> 37
| `Old_Italic -> 38
| `Gothic -> 39
| `Deseret -> 40
| `Tagalog -> 41
| `Hanunoo -> 42
| `Buhid -> 43
| `Tagbanwa -> 44

let script_tbl =
[|
`Common;
`Inherited;
`Latin;
`Greek;
`Cyrillic;
`Armenian;
`Hebrew;
`Arabic;
`Syriac;
`Thaana;
`Devanagari;
`Bengali;
`Gurmukhi;
`Gujarati;
`Oriya;
`Tamil;
`Telugu;
`Kannada;
`Malayalam;
`Sinhala;
`Thai;
`Lao;
`Tibetan;
`Myanmar;
`Georgian;
`Hangul;
`Ethiopic;
`Cherokee;
`Canadian_Aboriginal;
`Ogham;
`Runic;
`Khmer;
`Mongolian;
`Hiragana;
`Katakana;
`Bopomofo;
`Han;
`Yi;
`Old_Italic;
`Gothic;
`Deseret;
`Tagalog;
`Hanunoo;
`Buhid;
`Tagbanwa;
|]
type script_type = Script_type.t

let script_of_num i = script_tbl.(i)
let script_of_name = Script_type.script_type_of_name
let num_of_script = Script_type.num_of_script
let script_of_num = Script_type.script_of_num

type decomposition_type =
[ `Canon
Expand Down
47 changes: 1 addition & 46 deletions src/internal/unidata.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,52 +74,7 @@ module type Type = sig
val num_of_cat : general_category_type -> int
val cat_of_num : int -> general_category_type

type script_type =
[ `Common
| `Inherited
| `Latin
| `Greek
| `Cyrillic
| `Armenian
| `Hebrew
| `Arabic
| `Syriac
| `Thaana
| `Devanagari
| `Bengali
| `Gurmukhi
| `Gujarati
| `Oriya
| `Tamil
| `Telugu
| `Kannada
| `Malayalam
| `Sinhala
| `Thai
| `Lao
| `Tibetan
| `Myanmar
| `Georgian
| `Hangul
| `Ethiopic
| `Cherokee
| `Canadian_Aboriginal
| `Ogham
| `Runic
| `Khmer
| `Mongolian
| `Hiragana
| `Katakana
| `Bopomofo
| `Han
| `Yi
| `Old_Italic
| `Gothic
| `Deseret
| `Tagalog
| `Hanunoo
| `Buhid
| `Tagbanwa ]
type script_type = Script_type.t

val script_of_name : string -> script_type
val script_of_num : int -> script_type
Expand Down
Loading
Loading