Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 54 additions & 53 deletions tokenizers/tk-encode/src/models/bpe/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ use crate::models::bpe::At;
use crate::models::bpe::merge_hot_cold_queue::{Entry, MergeQueue, QueueScratch};
use crate::models::bpe::model::{Atoms, BPE};
use crate::models::bpe::tables::{BpeTables, ID_MASK, NO_SYMBOL, RANK_MASK};
use crate::vocab::bucket_vocab_store::BucketVocabStore;

/// Set only for the few models that decorate their atoms: `end_of_word_suffix` (CLIP, openai-gpt,
/// XLM) and `continuing_subword_prefix`. An atom's symbol then depends on its position in the
Expand Down Expand Up @@ -180,18 +179,25 @@ impl BPE {

fn convert<M: SinkMode>(&self, sequence: &str, sink: &mut SymbolSink<M>) {
if let Some(affixes) = &self.affixes {
convert_affixed(
&self.tables,
&self.vocab,
&self.atoms,
affixes,
sequence,
sink,
);
convert_affixed(self, affixes, sequence, sink);
} else {
match &self.atoms {
Atoms::Bytes => convert_bytes(&self.tables, sequence.as_bytes(), sink),
Atoms::Chars { .. } => convert_chars(&self.tables, &self.atoms, sequence, sink),
Atoms::Bytes {
missing_bytes: true,
} if self.unk_token.is_some() => {
let mut in_unk_run = false;
for &byte in sequence.as_bytes() {
let symbol = self.tables.byte_internal[byte as usize];
if symbol == NO_SYMBOL {
self.push_unk(&mut in_unk_run, sink);
} else {
in_unk_run = false;
sink.push(&self.tables, symbol);
}
}
}
Atoms::Bytes { .. } => convert_bytes(&self.tables, sequence.as_bytes(), sink),
Atoms::Chars { .. } => convert_chars(self, sequence, sink),
}
}
}
Expand Down Expand Up @@ -230,22 +236,17 @@ fn convert_bytes<M: SinkMode>(tables: &BpeTables, bytes: &[u8], sink: &mut Symbo

/// Character-level conversion, for models without a byte-level pretokenizer: every vocab token
/// of one character has a fold entry, so there is no byte decomposition to do here.
fn convert_chars<M: SinkMode>(
tables: &BpeTables,
atoms: &Atoms,
sequence: &str,
sink: &mut SymbolSink<M>,
) {
fn convert_chars<M: SinkMode>(model: &BPE, sequence: &str, sink: &mut SymbolSink<M>) {
let mut in_unk_run = false;
let sequence_bytes = sequence.as_bytes();
for (start, character) in sequence.char_indices() {
let symbol = tables.fold.get_char(character);
let symbol = model.tables.fold.get_char(character);
if symbol != NO_SYMBOL {
in_unk_run = false;
sink.push(tables, symbol);
sink.push(&model.tables, symbol);
} else {
let atom_bytes = &sequence_bytes[start..start + character.len_utf8()];
push_unknown(tables, atoms, &mut in_unk_run, atom_bytes, sink);
model.push_unknown(&mut in_unk_run, atom_bytes, sink);
}
}
}
Expand All @@ -256,9 +257,7 @@ fn convert_chars<M: SinkMode>(
/// form is assembled in a stack buffer and looked up in the vocab, which costs a hash per atom --
/// these models are rare enough that it is not worth a second fold table to avoid it.
fn convert_affixed<M: SinkMode>(
tables: &BpeTables,
vocab: &BucketVocabStore,
atoms: &Atoms,
model: &BPE,
affixes: &Affixes,
sequence: &str,
sink: &mut SymbolSink<M>,
Expand All @@ -268,8 +267,8 @@ fn convert_affixed<M: SinkMode>(
let mut in_unk_run = false;
let mut pos = 0;
while pos < sequence_bytes.len() {
let atom_len = match atoms {
Atoms::Bytes => 1,
let atom_len = match model.atoms {
Atoms::Bytes { .. } => 1,
Atoms::Chars { .. } => UTF8_LEN[sequence_bytes[pos] as usize] as usize,
};
let atom = &sequence_bytes[pos..pos + atom_len];
Expand All @@ -290,47 +289,49 @@ fn convert_affixed<M: SinkMode>(
len += suffix.len();
}

let symbol = vocab
let symbol = model
.vocab
.get_bytes(&buffer[..len])
.and_then(|external| affixes.to_internal.get(external as usize).copied())
.filter(|&symbol| symbol != NO_SYMBOL);
match symbol {
Some(symbol) => {
in_unk_run = false;
sink.push(tables, symbol);
sink.push(&model.tables, symbol);
}
None => push_unknown(tables, atoms, &mut in_unk_run, atom, sink),
None => model.push_unknown(&mut in_unk_run, atom, sink),
}
}
}

/// An atom missing from the vocab: its bytes if the model is byte-level or has `byte_fallback`,
/// else `unk`. `in_unk_run` is true while the atoms just before were all turned into `unk`.
#[inline(always)]
fn push_unknown<M: SinkMode>(
tables: &BpeTables,
atoms: &Atoms,
in_unk_run: &mut bool,
atom: &[u8],
sink: &mut SymbolSink<M>,
) {
match atoms {
Atoms::Bytes => sink.push_bytes(tables, &tables.byte_internal, atom),
Atoms::Chars {
byte_fallback,
unk_token,
fuse_unk,
} => {
if let Some(fallback) = byte_fallback {
sink.push_bytes(tables, fallback, atom);
impl BPE {
fn push_unknown<M: SinkMode>(
&self,
in_unk_run: &mut bool,
atom: &[u8],
sink: &mut SymbolSink<M>,
) {
match &self.atoms {
Atoms::Chars {
byte_fallback: Some(fallback),
} => {
sink.push_bytes(&self.tables, fallback, atom);
*in_unk_run = false;
} else if let Some(unk) = unk_token {
// with fuse_unk the run already emitted its unk, so this atom adds nothing
if !(*fuse_unk && *in_unk_run) {
sink.push(tables, *unk);
}
*in_unk_run = true;
}
Atoms::Bytes { .. } if self.unk_token.is_none() => {
sink.push_bytes(&self.tables, &self.tables.byte_internal, atom);
}
_ => self.push_unk(in_unk_run, sink),
}
}

#[inline(always)]
fn push_unk<M: SinkMode>(&self, in_unk_run: &mut bool, sink: &mut SymbolSink<M>) {
if let Some(unk) = self.unk_token {
if !(self.fuse_unk && *in_unk_run) {
sink.push(&self.tables, unk);
}
*in_unk_run = true;
}
}
}
18 changes: 12 additions & 6 deletions tokenizers/tk-encode/src/models/bpe/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ const _: () = assert!(align_of::<Token>() == align_of::<u32>());

pub struct BPE {
pub(super) atoms: Atoms,
pub(super) unk_token: Option<u32>,
pub(super) fuse_unk: bool,
pub(super) tables: BpeTables,
pub(super) affixes: Option<Affixes>,
pub(super) vocab: BucketVocabStore,
Expand All @@ -65,11 +67,11 @@ pub struct BPE {
#[allow(clippy::large_enum_variant)]
pub(super) enum Atoms {
/// The atoms are the 256 bytes; the symbol for each lives in `BpeTables::byte_internal`.
Bytes,
Bytes {
missing_bytes: bool,
},
Chars {
byte_fallback: Option<[u32; 256]>,
unk_token: Option<u32>,
fuse_unk: bool,
},
}

Expand All @@ -79,7 +81,7 @@ impl BPE {
/// **decoded raw bytes** at load time. Decoding is then a concatenation, and running a
/// `ByteLevel` decoder over these entries would decode a second time.
pub fn is_byte_level(&self) -> bool {
matches!(self.atoms, Atoms::Bytes)
matches!(self.atoms, Atoms::Bytes { .. })
}

/// A token's bytes, borrowed from the vocab store's slab. For a byte-level model these are
Expand Down Expand Up @@ -146,8 +148,12 @@ impl BPE {
let Ok(text) = std::str::from_utf8(bytes) else {
continue;
};
let chars = text.chars().count();
let foldable = if chars == 0 || (chars == 1 && !has_suffix) {
let single_atom = if self.is_byte_level() {
bytes.len() == 1
} else {
text.chars().count() == 1
};
let foldable = if bytes.is_empty() || (single_atom && !has_suffix) {
// A single atom has no pair to merge and is trivially its own encoding.
true
} else {
Expand Down
49 changes: 23 additions & 26 deletions tokenizers/tk-encode/src/models/bpe/serialization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,20 +106,13 @@ impl BPE {
merges.push((token(left)?, token(right)?));
}

let (unk_token, fuse_unk, byte_fallback) = match &self.atoms {
Atoms::Bytes => (None, false, false),
let unk_token = self.unk_token.map(token).transpose()?;
let byte_fallback = matches!(
self.atoms,
Atoms::Chars {
unk_token,
fuse_unk,
byte_fallback,
} => {
let unk = match *unk_token {
Some(internal) => Some(token(internal)?),
None => None,
};
(unk, *fuse_unk, byte_fallback.is_some())
byte_fallback: Some(_)
}
};
);
let (prefix, suffix): (&[u8], &[u8]) = match &self.affixes {
Some(affixes) => (&affixes.prefix, &affixes.suffix),
None => (&[], &[]),
Expand All @@ -132,7 +125,7 @@ impl BPE {
unk_token,
continuing_subword_prefix: affix(prefix)?,
end_of_word_suffix: affix(suffix)?,
fuse_unk,
fuse_unk: self.fuse_unk,
byte_fallback,
ignore_merges: self.vocab.all_foldable(),
dropout: None,
Expand Down Expand Up @@ -224,35 +217,39 @@ impl BPE {
.copied()
.filter(|&internal| internal != NO_SYMBOL)
};
let unk_token = unk_token
.map(|name| {
vocab
.get(&name)
.copied()
.and_then(to_internal)
.ok_or(Error::UnkTokenOutOfVocabulary(name))
})
.transpose()?;
let (vocab, atoms) = if byte_level {
let missing: Vec<u8> = VALID_UTF8_BYTES
.into_iter()
.flatten()
.filter(|&byte| tables.byte_internal[byte as usize] == NO_SYMBOL)
.collect();
if !missing.is_empty() {
if !missing.is_empty() && unk_token.is_none() {
warn!(
"this byte-level vocabulary has no token for bytes {missing:02X?}: they will be \
ignored at encoding."
);
}
(byte_level::transform_vocab(vocab)?, Atoms::Bytes)
let missing_bytes = tables.byte_internal.contains(&NO_SYMBOL);
(
byte_level::transform_vocab(vocab)?,
Atoms::Bytes { missing_bytes },
)
} else {
let vocab = BucketVocabStore::build(
vocab
.into_iter()
.map(|(k, v)| (k.into_bytes(), v))
.collect(),
)?;
let unk_token = if let Some(unk_str) = unk_token {
let token_id = vocab
.token_to_id(&unk_str)
.ok_or_else(|| Error::UnkTokenOutOfVocabulary(unk_str.clone()))?;
Some(token_id)
} else {
None
};
let unk_token = unk_token.map(|external| to_internal(external).unwrap_or(NO_SYMBOL));
let fallback_lookup = if byte_fallback {
let mut fallback_lookup = [NO_SYMBOL; 256];
for b in 0u8..=255 {
Expand Down Expand Up @@ -298,8 +295,6 @@ impl BPE {
(
vocab,
Atoms::Chars {
fuse_unk,
unk_token,
byte_fallback: fallback_lookup,
},
)
Expand All @@ -319,6 +314,8 @@ impl BPE {
});
let mut built = Self {
atoms,
unk_token,
fuse_unk,
tables,
affixes,
cache_capacity,
Expand Down
51 changes: 51 additions & 0 deletions tokenizers/tk-encode/src/models/bpe/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -653,3 +653,54 @@ fn byte_level_drops_bytes_real_text_can_contain() {
let want: Vec<u32> = "ab".repeat(64).bytes().map(u32::from).collect();
assert_eq!(pipeline_ids(&pipeline, &long), want);
}

#[test]
fn byte_level_emits_unknowns_without_merging_their_neighbors() {
for fuse_unk in [false, true] {
let model = BPE::from_config(BpeConfig {
vocab: v(&[("<unk>", 0), ("a", 1), ("b", 2), ("ab", 3)]),
merges: m(&[("a", "b")]),
byte_level: true,
unk_token: Some("<unk>".into()),
fuse_unk,
..Default::default()
})
.unwrap();
assert_eq!(pipeline_ids(&model, "a\rb"), [1, 0, 2]);
let word = "aéb\rab";
let expected = if fuse_unk {
vec![1, 0, 2, 0, 3]
} else {
vec![1, 0, 0, 2, 0, 3]
};
for count in [1, 64] {
assert_eq!(
pipeline_ids(&model, &word.repeat(count)),
expected.repeat(count)
);
}
let mut config = model.to_config().unwrap();
assert_eq!(config.unk_token.as_deref(), Some("<unk>"));
assert_eq!(config.fuse_unk, fuse_unk);

// The final atom is looked up as b</w>. The bare b exists, but the suffixed one doesn't.
config.end_of_word_suffix = Some("</w>".into());
let affixed = BPE::from_config(config).unwrap();
let expected = if fuse_unk { vec![1, 0] } else { vec![1, 0, 0] };
assert_eq!(pipeline_ids(&affixed, "a\rb"), expected);
}
}

#[test]
fn byte_level_word_hit_still_checks_unknown_atoms() {
// The vocabulary has a token for the UTF-8 spelling of é, but no atom
// for its second byte, so the merge engine cannot produce that token.
let model = BPE::from_config(BpeConfig {
vocab: v(&[("<unk>", 0), ("Ã", 1), ("é", 2)]),
byte_level: true,
unk_token: Some("<unk>".into()),
..Default::default()
})
.unwrap();
assert_eq!(pipeline_ids(&model, "é"), [1, 0]);
}