Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion bindings/node/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion bindings/python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion bindings/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ dev = [
"maturin>=1.0,<2.0",
"pytest",
"ruff",
"ty",
"ty==0.0.102",
]

[build-system]
Expand Down
10 changes: 5 additions & 5 deletions bindings/python/tests/test_composition.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def test_editing_a_component_writes_through(gpt2_spec):
def test_sequence_setitem(gpt2_spec):
gpt2_spec.normalizer = normalizers.Sequence([normalizers.Prepend("a")])
gpt2_spec.normalizer[0] = normalizers.Prepend("b")
assert gpt2_spec.normalizer[0].prepend == "b" # ty: ignore[unresolved-attribute]
assert gpt2_spec.normalizer[0].prepend == "b" # ty: ignore[missing-attribute]


# Each kind lists a Sequence's items under its own key, and "pretokenizers" has no underscore.
Expand Down Expand Up @@ -303,21 +303,21 @@ def test_sequence_item_is_shared():
item = sequence[0]
assert isinstance(item, normalizers.Prepend)
item.prepend = "b"
assert sequence[0].prepend == "b" # ty: ignore[unresolved-attribute]
assert sequence[0].prepend == "b" # ty: ignore[missing-attribute]


def test_sequence_shares_its_arguments():
prepend = normalizers.Prepend("a")
sequence = normalizers.Sequence([prepend])
prepend.prepend = "b"
assert sequence[0].prepend == "b" # ty: ignore[unresolved-attribute]
assert sequence[0].prepend == "b" # ty: ignore[missing-attribute]


def test_sequence_negative_index():
sequence = normalizers.Sequence([normalizers.Prepend("a"), normalizers.Prepend("b")])
assert sequence[-1].__getstate__() == sequence[1].__getstate__()
sequence[-2] = normalizers.Prepend("c")
assert [item.prepend for item in sequence] == ["c", "b"] # ty: ignore[unresolved-attribute]
assert [item.prepend for item in sequence] == ["c", "b"] # ty: ignore[missing-attribute]


def test_sequence_index_out_of_range():
Expand Down Expand Up @@ -345,7 +345,7 @@ def test_sequence_flattens_a_sequence_argument():
item = inner[0]
assert isinstance(item, normalizers.Prepend)
item.prepend = "x"
assert outer[0].prepend == "x" # ty: ignore[unresolved-attribute]
assert outer[0].prepend == "x" # ty: ignore[missing-attribute]


# A file can nest a Sequence in a Sequence.
Expand Down
2 changes: 1 addition & 1 deletion bindings/python/tests/test_threads.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,4 @@ def decode():
# `sys._is_gil_enabled` exists on every build from 3.13 on, so only the config var tells the two apart.
@pytest.mark.skipif(not sysconfig.get_config_var("Py_GIL_DISABLED"), reason="only meaningful on a free-threaded build")
def test_gil_disabled():
assert sys._is_gil_enabled() is False # ty: ignore[unresolved-attribute]
assert sys._is_gil_enabled() is False # ty: ignore[missing-attribute]
1 change: 0 additions & 1 deletion bindings/wasm/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 10 additions & 13 deletions tokenizers/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion tokenizers/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ authors = [
edition = "2024"
name = "tokenizers"
version = "1.0.0-dev.2"
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/tokenizers/"
license = "Apache-2.0"
Expand Down
1 change: 0 additions & 1 deletion tokenizers/bitcannon/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ authors = [
"Luc Georges <luc@huggingface.co>",
"Simon Brandeis <simon@huggingface.co>",
]
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/bitcannon/"
license = "Apache-2.0"
Expand Down
9 changes: 8 additions & 1 deletion tokenizers/bitcannon/src/classes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,14 @@ const fn lut<const DROP: u16, const ISOLATE: u16, const KEEP_A: u16>() -> [u8; 6
streams!(
/// One block's class streams — one per dense code. Only `other` needs the `valid` mask, since
/// past the block end every plane reads 0, i.e. code 0.
Cls { lead, cont, dropped, isolate, keep_a, other }
Cls {
lead,
cont,
dropped,
isolate,
keep_a,
other
}
);

/// Decode the two planes; returns the fill seed for the block after.
Expand Down
8 changes: 4 additions & 4 deletions tokenizers/bitcannon/src/classify/atom_tables.rs

Large diffs are not rendered by default.

12 changes: 11 additions & 1 deletion tokenizers/bitcannon/src/models/family_gpt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,17 @@ pub(crate) const LUT: [u8; 64] = {
streams!(
/// One block's class streams. `other` folds the apostrophe code back in; only `l` needs the
/// `valid` mask, since past the block end every plane reads 0, i.e. code 0.
Cls { lead, cont, l, n, other, nl, sp, ws, apo }
Cls {
lead,
cont,
l,
n,
other,
nl,
sp,
ws,
apo
}
);

/// Build one block's streams; returns the fill seed for the block after.
Expand Down
19 changes: 18 additions & 1 deletion tokenizers/bitcannon/src/models/family_o200k.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,24 @@ streams!(
/// own code, so it is not in `u`/`l`/`c` to begin with (kimi's classes are literally
/// `[…&&[^\p{Han}]]`). `han` is every Han byte; `dual` the ones ALSO in `oth` / `n`. `slash` is
/// the only remaining AUX stream.
Cls { lead, cont, u, l, c, mark, n, nl, sp, ws, oth, apo, letter, slash, han, dual }
Cls {
lead,
cont,
u,
l,
c,
mark,
n,
nl,
sp,
ws,
oth,
apo,
letter,
slash,
han,
dual
}
);

/// Cross-block state that is not a stream.
Expand Down
10 changes: 3 additions & 7 deletions tokenizers/tk-convert/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ authors = [
"Luc Georges <luc@huggingface.co>",
"Simon Brandeis <simon@huggingface.co>",
]
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/tokenizers/"
license = "Apache-2.0"
Expand All @@ -33,12 +32,9 @@ thiserror = "2"
# the FSM fast path without changing any test.
bitcannon = { path = "../bitcannon", version = "0.1.0-dev.2" }

# Latest released tokenizers, the independent id/decode oracle `tests/oracle.rs` compares against.
tokenizers-release = { package = "tokenizers", version = "=0.23.2", optional = true }

[features]
default = []
bench-baseline = ["dep:tokenizers-release"]
bench-baseline = []

[dev-dependencies]
# No version pins on these two: cargo strips path dev-deps when packaging, and tk-serialize
Expand All @@ -59,5 +55,5 @@ tk-encode = { path = "../tk-encode", default-features = false, features = [
# The oracle's long case must reach the parallel encoder, not the serial path.
"parallelism",
] }
criterion = "0.8"
rayon = "1.12"
# Latest released tokenizers, the independent id/decode oracle `tests/oracle.rs` compares against.
tokenizers-release = { package = "tokenizers", version = "=0.23.2" }
1 change: 0 additions & 1 deletion tokenizers/tk-encode/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ authors = [
"Luc Georges <luc@huggingface.co>",
"Simon Brandeis <simon@huggingface.co>",
]
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/tokenizers/"
license = "Apache-2.0"
Expand Down
4 changes: 2 additions & 2 deletions tokenizers/tk-encode/src/models/unigram/lattice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -376,8 +376,8 @@ impl<'a> Lattice<'a> {

let eos_id = self.begin_nodes[len][0].borrow().node_id;
let z = alpha[eos_id];
for pos in 0..len {
for node in &self.begin_nodes[pos] {
for begin_nodes in self.begin_nodes.iter().take(len) {
for node in begin_nodes {
let node_id = node.borrow().node_id;
let id = node.borrow().id;
let a = alpha[node_id];
Expand Down
4 changes: 2 additions & 2 deletions tokenizers/tk-encode/src/utils/word_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ impl<'a> WordCache {
if slot.key == placement.key && !slot.is_spilled() {
// SAFETY: the caller guarantees room for `MAX_INLINE_IDS`. Lanes past `ids_len` are
unsafe {
for lane in 0..MAX_INLINE_IDS {
dst.add(lane).write(slot.payload[lane]);
for (lane, &payload) in slot.payload.iter().enumerate().take(MAX_INLINE_IDS) {
dst.add(lane).write(payload);
}
}
return ProbeEmit::Wrote(slot.ids_len as usize);
Expand Down
1 change: 0 additions & 1 deletion tokenizers/tk-serialize/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ authors = [
"Luc Georges <luc@huggingface.co>",
"Simon Brandeis <simon@huggingface.co>",
]
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/tokenizers/"
license = "Apache-2.0"
Expand Down
2 changes: 0 additions & 2 deletions tokenizers/tk-train/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ authors = [
"Luc Georges <luc@huggingface.co>",
"Simon Brandeis <simon@huggingface.co>",
]
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/tokenizers/"
license = "Apache-2.0"
Expand All @@ -26,7 +25,6 @@ tk-encode = { path = "../tk-encode", version = "0.1.0-dev.2", default-features =
"parallelism",
] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
ahash = { version = "0.8.11", features = ["serde"] }
compact_str = { version = "0.9", features = ["serde"] }
dary_heap = { version = "0.3.6", features = ["serde"] }
Expand Down
6 changes: 1 addition & 5 deletions tokenizers/tk-train/src/trainers/unigram.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,11 +180,7 @@ impl UnigramTrainer {
// care of within pre_tokenizers.
// https://github.com/google/sentencepiece/blob/26be9516cd81d5315ee31c48d2438018e0eab879/src/trainer_interface.cc#L203
let n = char_string.len();
if char_string.is_empty() || n > self.max_piece_length {
return false;
}

true
!(char_string.is_empty() || n > self.max_piece_length)
}

fn finalize(&self, model: Unigram, required_chars: AHashSet<String>) -> Result<Unigram> {
Expand Down
Loading