Repository: zurawiki/tiktoken-rs (language: rust)

Issue:
Incomplete utf-8 byte sequence from index 0
The code is:
```
#[test]
fn test_token() {
    let input = "🍌This is a sentence   with spaces, hahhahah haha ha";
    let rke = r50k_base()?;
    let _ = rke.split_by_token(input, true).unwrap();
}
```

Error:
```
called `Result::unwrap()` on an `Err` value: incomplete utf-8 byte sequence from index 0
```

Caused by this line:
https://github.com/zurawiki/tiktoken-rs/blob/1aa3d90f220ec4dc42a2ff489a42b75b4e3b6cf8/tiktoken-rs/src/vendor_tiktoken.rs#L618

It should be `String::from_utf8_lossy`?


Discussion / hints:

Fix patch:
diff --git a/tiktoken-rs/src/vendor_tiktoken.rs b/tiktoken-rs/src/vendor_tiktoken.rs
index 919e6fe..28b0515 100644
--- a/tiktoken-rs/src/vendor_tiktoken.rs
+++ b/tiktoken-rs/src/vendor_tiktoken.rs
@@ -613,10 +613,10 @@ impl CoreBPE {
             false => self.encode_ordinary(text),
         };
 
-        self._decode_native_and_split(encoded).map(|token|
-                // Map each token to a Result<String>
-                String::from_utf8(token)
-                .map_err(|e| anyhow!(e.to_string())))
+        self._decode_native_and_split(encoded).map(|token| {
+            // Map each token to a Result<String>
+            Ok(String::from_utf8_lossy(token.as_slice()).to_string())
+        })
     }
 
     /// Tokenize a string and return the decoded tokens using the correct BPE model.


Test patch:
diff --git a/tiktoken-rs/tests/tiktoken.rs b/tiktoken-rs/tests/tiktoken.rs
index 9efeac6..988fe39 100644
--- a/tiktoken-rs/tests/tiktoken.rs
+++ b/tiktoken-rs/tests/tiktoken.rs
@@ -121,3 +121,24 @@ fn p50k_base_singleton_test() {
     }
     // println!("p50k_base encode/decode 2: {:?}", now.elapsed());
 }
+
+#[test]
+fn test_unicode_encode() {
+    let bpe = r50k_base().unwrap();
+
+    let input = "🍌This is a sentence";
+    let tokenized = bpe.split_by_token(input, true).unwrap();
+    assert_eq!(tokenized.len(), 7);
+
+    let input = "你会说中文吗？";
+    let tokenized = bpe.split_by_token(input, true).unwrap();
+    assert_eq!(tokenized.len(), 14);
+}
+
+#[test]
+fn test_unicode_roundtrip() {
+    test_roundtrip(&cl100k_base().unwrap(), "🍌This is a sentence");
+    test_roundtrip(&p50k_base().unwrap(), "我想借几本汉语书");
+    test_roundtrip(&r50k_base().unwrap(), "我想借几本汉语书");
+    test_roundtrip(&cl100k_base().unwrap(), "你会说中文吗？");
+}