Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
974 changes: 536 additions & 438 deletions .github/scripts/render_pipeline_bench.py

Large diffs are not rendered by default.

231 changes: 167 additions & 64 deletions .github/workflows/pipeline-bench.yml

Large diffs are not rendered by default.

3 changes: 0 additions & 3 deletions tokenizers/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 12 additions & 2 deletions tokenizers/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,18 @@ SHARED_RESOURCES = $(DATA_DIR)/gpt2-vocab.json $(DATA_DIR)/gpt2-merges.txt $(DAT
BENCHMARK_RESOURCES = $(SHARED_RESOURCES) $(DATA_DIR)/unigram_wagahaiwa_nekodearu.txt $(DATA_DIR)/bert-wiki.json $(DATA_DIR)/fixtures/modalities/agentic-traces.txt
TESTS_RESOURCES = $(SHARED_RESOURCES) $(DATA_DIR)/unigram.json $(DATA_DIR)/unigram_wagahaiwa_nekodearu.txt $(DATA_DIR)/roberta.json $(DATA_DIR)/tokenizer-wiki.json $(DATA_DIR)/bert-wiki.json $(DATA_DIR)/fixtures/modalities/agentic-traces.txt

FIXTURE_LANGS = amh_Ethi arb_Arab ben_Beng cmn_Hani ell_Grek eng_Latn heb_Hebr hin_Deva jpn_Jpan kat_Geor kor_Hang rus_Cyrl tam_Taml tha_Thai
FIXTURE_MODALITIES = agentic_swe agentic-traces code_mixed math_latex added_special_sparse added_special_dense added_normalized_sparse added_normalized_dense
# One language per encode regime (script structure × vocab coverage), not one per
# script: eng is the vocab-native anchor, rus the 2-byte European alphabet, arb the
# 2-byte RTL abjad, hin mainstream Indic, tam the extreme-fertility stress probe,
# amh the poorly-covered 3-byte script, cmn the Han inventory, jpn script mixing,
# tha spaceless text. Structural near-twins (ell/heb/ben/kat/kor) were cut: they
# tracked their representative on every model and only diluted the aggregates.
FIXTURE_LANGS = amh_Ethi arb_Arab cmn_Hani eng_Latn hin_Deva jpn_Jpan rus_Cyrl tam_Taml tha_Thai
# Only the sparse added-token corpora: realistic marker density (text sprinkled with
# specials, like chat-template traffic), covering both the raw-special and the
# normalized-added scan paths. The dense variants were marker soup that skewed every
# aggregate while measuring nothing the sparse ones don't.
FIXTURE_MODALITIES = agentic_swe agentic-traces code_mixed math_latex added_special_sparse added_normalized_sparse
FIXTURES_RESOURCES = $(FIXTURE_LANGS:%=$(DATA_DIR)/fixtures/lang/%.txt) $(FIXTURE_MODALITIES:%=$(DATA_DIR)/fixtures/modalities/%.txt)

# Models benched by tk-encode/examples/fixture_bench.rs; the manifest is the
Expand Down
17 changes: 2 additions & 15 deletions tokenizers/tk-encode/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,11 @@ memchr = "2.8.2"
unicode-normalization = "0.1.25"
yada = "0.7.0"
libc = "0.2"
wide = "1.6.0"

# Latest released tokenizers, used as the comparison baseline by the CI benchmark
# (examples gated on `bench-baseline`). Optional so production builds never pull it.
tokenizers-release = { package = "tokenizers", version = "=0.23.1", optional = true }
# Reference regex engines `fixture_bench` times the classify+fsm pre-tokenize against: Oniguruma and
# PCRE2 (both C) alongside the pure-Rust `fancy-regex` (optional, above). All three are pulled in by
# `bench-baseline` — optional + behind that feature so the C builds happen ONLY for the CI benchmark,
# never for `cargo test`/production.
onig = { version = "6.5.1", optional = true }
pcre2 = { version = "0.2", optional = true }
logos = { version = "0.15", optional = true } # compile-time DFA lexer reference (pure Rust)
wide = "1.6.0"

[features]
# `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a *regex* pattern in a `Split`
Expand All @@ -84,13 +77,7 @@ progressbar = ["indicatif"]
http = ["hf-hub"]
unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
rustls-tls = ["hf-hub?/rustls-tls"]
bench-baseline = [
"dep:tokenizers-release",
"dep:onig",
"dep:pcre2",
"dep:logos",
"fancy-regex",
]
bench-baseline = ["dep:tokenizers-release", "fancy-regex"]
parallelism = ["dep:rayon", "dep:rayon-cond"]

[dev-dependencies]
Expand Down
Loading
Loading