Repository: PyThaiNLP/nlpo3 (language: python)

Issue:
Consider remove optional out of safe and parallel flag in Tokenizer trait
From [this line](https://github.com/PyThaiNLP/nlpo3/blob/main/src/tokenizer/newmm_custom.rs#L368-L372), I think we can remove `Option` from `safe` and `parallel` to be just a boolean. The code should looks simpler and the binding just call [unwrap_or](https://doc.rust-lang.org/std/option/enum.Option.html#method.unwrap_or) or [unwrap_or_default](https://doc.rust-lang.org/std/option/enum.Option.html#method.unwrap_or_default) to convert option type to a boolean.


Discussion / hints:
I've some prototype idea in https://github.com/PyThaiNLP/nlpo3/pull/32. If you guys are all ok, I can continue fix on binding lib.

Fix patch:
diff --git a/src/tokenizer/newmm_custom.rs b/src/tokenizer/newmm_custom.rs
index 11f0b20..4e6ef8c 100644
--- a/src/tokenizer/newmm_custom.rs
+++ b/src/tokenizer/newmm_custom.rs
@@ -395,27 +395,11 @@ impl Newmm {
 }
 
 impl Tokenizer for Newmm {
-    fn segment(
-        &self,
-        text: &str,
-        safe: Option<bool>,
-        parallel: Option<bool>,
-    ) -> AnyResult<Vec<String>> {
-        let safe_flag = safe.unwrap_or(false);
-        let parallel_flag = match parallel {
-            Some(val) => val,
-            _ => false,
-        };
-        let custom_string = CustomString::new(text);
-        Self::internal_segment(&custom_string, &self.dict, safe_flag, parallel_flag)
+    fn segment(&self, text: &str, safe: bool, parallel: bool) -> AnyResult<Vec<String>> {
+        Self::internal_segment(&CustomString::new(text), &self.dict, safe, parallel)
     }
 
-    fn segment_to_string(
-        &self,
-        text: &str,
-        safe: Option<bool>,
-        parallel: Option<bool>,
-    ) -> Vec<String> {
+    fn segment_to_string(&self, text: &str, safe: bool, parallel: bool) -> Vec<String> {
         self.segment(text, safe, parallel).unwrap()
     }
 }
diff --git a/src/tokenizer/tokenizer_trait.rs b/src/tokenizer/tokenizer_trait.rs
index a652590..59cdba9 100644
--- a/src/tokenizer/tokenizer_trait.rs
+++ b/src/tokenizer/tokenizer_trait.rs
@@ -1,16 +1,7 @@
 use anyhow::Result as AnyResult;
+
 pub trait Tokenizer {
-    fn segment(
-        &self,
-        text: &str,
-        safe: Option<bool>,
-        parallel: Option<bool>,
-    ) -> AnyResult<Vec<String>>;
+    fn segment(&self, text: &str, safe: bool, parallel: bool) -> AnyResult<Vec<String>>;
 
-    fn segment_to_string(
-        &self,
-        text: &str,
-        safe: Option<bool>,
-        parallel: Option<bool>,
-    ) -> Vec<String>;
+    fn segment_to_string(&self, text: &str, safe: bool, parallel: bool) -> Vec<String>;
 }


Test patch:
diff --git a/tests/test_tokenizer.rs b/tests/test_tokenizer.rs
index 006d606..2faa6f7 100644
--- a/tests/test_tokenizer.rs
+++ b/tests/test_tokenizer.rs
@@ -150,9 +150,9 @@ fn test_long_text_byte_tokenizer() {
     .join("");
 
     let newmm_default_dict = NewmmCustom::new(None);
-    let result = newmm_default_dict.segment(&long_text, None, Some(true)).unwrap();
+    let result = newmm_default_dict.segment(&long_text, false, true).unwrap();
 
-    let safe_result = newmm_default_dict.segment(&long_text, Some(true), Some(true)).unwrap();
+    let safe_result = newmm_default_dict.segment(&long_text, true, true).unwrap();
     assert_eq!(result.len(), 1889);
     assert_eq!(safe_result.len(), 1991);
 }
@@ -160,27 +160,27 @@ fn test_long_text_byte_tokenizer() {
 fn test_standard_short_word() {
     let newmm_default_dict = NewmmCustom::new(None);
     assert_eq!(
-        newmm_default_dict.segment_to_string("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", None, None),
+        newmm_default_dict.segment_to_string("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", false, false),
         ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("19...", None, None),
+        newmm_default_dict.segment_to_string("19...", false, false),
         ["19", "..."]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("19.", None, None),
+        newmm_default_dict.segment_to_string("19.", false, false),
         ["19", "."]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("19.84", None, None),
+        newmm_default_dict.segment_to_string("19.84", false, false),
         ["19.84"]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("127.0.0.1", None, None),
+        newmm_default_dict.segment_to_string("127.0.0.1", false, false),
         ["127.0.0.1"]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("USD1,984.42", None, None),
+        newmm_default_dict.segment_to_string("USD1,984.42", false, false),
         ["USD", "1,984.42"]
     );
 }
@@ -188,11 +188,11 @@ fn test_standard_short_word() {
 fn test_with_some_real_data() {
     let newmm_default_dict = NewmmCustom::new(None);
     assert_eq!(
-        newmm_default_dict.segment_to_string(FIRST_TEXT, None, None),
+        newmm_default_dict.segment_to_string(FIRST_TEXT, false, false),
         ["นิสสัน", "ผ่อน", "จน", "เพลีย", "นาวา", "ร่า", ".."]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string(SECOND_TEXT, None, None),
+        newmm_default_dict.segment_to_string(SECOND_TEXT, false, false),
         [
             "อาชญากรรม",
             "ทางการแพทย์",
@@ -226,23 +226,23 @@ fn test_with_some_real_data() {
 fn test_thai_number() {
     let newmm_default_dict = NewmmCustom::new(None);
     assert_eq!(
-        newmm_default_dict.segment_to_string("๑๙...", None, None),
+        newmm_default_dict.segment_to_string("๑๙...", false, false),
         ["๑๙", "..."]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("๑๙.", None, None),
+        newmm_default_dict.segment_to_string("๑๙.", false, false),
         ["๑๙", "."]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("๑๙.๘๔", None, None),
+        newmm_default_dict.segment_to_string("๑๙.๘๔", false, false),
         ["๑๙.๘๔"]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("๑๒๗.๐.๐.๑", None, None),
+        newmm_default_dict.segment_to_string("๑๒๗.๐.๐.๑", false, false),
         ["๑๒๗.๐.๐.๑"]
     );
     assert_eq!(
-        newmm_default_dict.segment_to_string("USD๑,๙๘๔.๔๒", None, None),
+        newmm_default_dict.segment_to_string("USD๑,๙๘๔.๔๒", false, false),
         ["USD", "๑,๙๘๔.๔๒"]
     );
 }