Return ValueError instead of panicking on duplicate mergeable_ranks

src/lib.rs

@@ -633,12 +633,14 @@ impl CoreBPE {
         let decoder: HashMap<Rank, Vec<u8>> =
             encoder.iter().map(|(k, v)| (*v, k.clone())).collect();
 
-        assert!(
-            encoder.len() == decoder.len(),
-            "Encoder and decoder must be of equal length. Encoder length: {}, decoder length: {}.\nMaybe you had duplicate token indices in your encoder?",
-            encoder.len(),
-            decoder.len()
-        );
+        if encoder.len() != decoder.len() {
+            return Err(format!(
+                "Encoder and decoder must be of equal length. Encoder length: {}, decoder length: {}.\nMaybe you had duplicate token indices in your encoder?",
+                encoder.len(),
+                decoder.len()
+            )
+            .into());
+        }
 
         let special_tokens_decoder: HashMap<Rank, Vec<u8>> = special_tokens_encoder
             .iter()

tests/test_duplicate_ranks.py

@@ -0,0 +1,13 @@
+import pytest
+
+import tiktoken
+
+
+def test_duplicate_mergeable_ranks_raise_value_error():
+    with pytest.raises(ValueError):
+        tiktoken.Encoding(
+            name="duplicate_ranks",
+            pat_str=r".",
+            mergeable_ranks={b"a": 0, b"b": 0, b"c": 1},
+            special_tokens={},
+        )