Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions tokenizers/tk-encode/src/models/wordpiece/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ impl pipeline::ModelScratch for WordPieceScratch {}

pub struct PipelineWordPiece {
vocab_trie: yada::DoubleArray<Vec<u8>>,
vocab_r: Box<[Option<Box<str>>]>,
unk_token: Option<u32>,
continuing_subword_prefix: String,
max_input_chars_per_word: usize,
Expand All @@ -347,12 +348,18 @@ impl TryFrom<WordPiece> for PipelineWordPiece {
let mut keyset: Vec<_> = vocab.into_iter().collect();
keyset.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
let vocab_trie = DoubleArray::new(DoubleArrayBuilder::build(&keyset)?)?;
let max_id = keyset.iter().map(|&(_, id)| id).max().unwrap_or(0) as usize;
let mut vocab_r = vec![None; max_id + 1];
for (token, id) in keyset {
vocab_r[id as usize] = Some(token.into_boxed_str());
}

Ok(Self {
continuing_subword_prefix,
max_input_chars_per_word,
unk_token,
vocab_trie,
vocab_r: vocab_r.into_boxed_slice(),
})
}
}
Expand Down Expand Up @@ -408,6 +415,10 @@ impl PipelineWordPiece {
}
Ok(())
}

pub fn id_to_token(&self, id: u32) -> Option<String> {
self.vocab_r.get(id as usize)?.as_deref().map(str::to_owned)
}
}

impl pipeline::Model for PipelineWordPiece {
Expand Down
11 changes: 1 addition & 10 deletions tokenizers/tk-encode/src/tokenizer/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -897,11 +897,6 @@ impl PipelineTokenizer {
{
return Ok(self.decode_byte_level(bpe, ids, skip_special_tokens));
}
if matches!(self.model, PipelineModel::WordPiece(_)) {
// Fail loud instead of dropping every id and returning "".
return Err("PipelineTokenizer::decode: WordPiece keeps no id -> token map yet".into());
}

let tokens = ids
.iter()
.filter_map(|&id| {
Expand Down Expand Up @@ -1344,16 +1339,12 @@ pub enum PipelineModel {

impl PipelineModel {
/// `id -> token`, for the decoder-chain route in [`PipelineTokenizer::decode`].
///
/// `None` for [`PipelineWordPiece`], which keeps only the forward `vocab_trie` and so has no
/// id -> token direction to answer with. `decode` refuses that model up front rather than
/// letting every id drop out and returning an empty string.
fn id_to_token(&self, id: u32) -> Option<String> {
match self {
Self::BPE(model) => model.id_to_token(id),
Self::Unigram(model) => model.id_to_token(id),
Self::WordLevel(model) => model.id_to_token(id),
Self::WordPiece(_) => None,
Self::WordPiece(model) => model.id_to_token(id),
}
}
}
Expand Down
Loading