Repository: PyThaiNLP/nlpo3 (language: python)

Issue:
Remove dictionary from default install
Current dictionary is quite big and probably not every users are going to use that.

Also, PyThaiNLP may consider to include nlpO3 Python-binding as one of the word tokenzation options. That means two exact copies of a same dictionary will be installed.

Dictionary can be installed if the user explicitly put it in a feature flag, maybe something like:
```
nlpo3 = {version = "1.3.0", features = ["dict-pythainlp", "dict-libthai"]}
```

More dictionary options can be offered in this way as well.

If user just install it without the feature flag:
```
nlpo3 = {version = "1.3.0"}
```
the user have to supply a dictionary of their own.


Discussion / hints:

Fix patch:
diff --git a/nlpo3-nodejs/src/lib.rs b/nlpo3-nodejs/src/lib.rs
index f7b8e22..cf0c7c9 100644
--- a/nlpo3-nodejs/src/lib.rs
+++ b/nlpo3-nodejs/src/lib.rs
@@ -12,9 +12,7 @@ fn load_dict(mut cx: FunctionContext) -> JsResult<JsString> {
     let mut dict_col_lock = DICT_COLLECTION.lock().unwrap();
     let file_path = cx.argument::<JsString>(0)?.value(&mut cx);
     let dict_name = cx.argument::<JsString>(1)?.value(&mut cx);
-    if dict_name == "default" {
-        Ok(cx.string(format!("Failed: 'default' dictionary name is reserved")))
-    } else if let Some(_) = dict_col_lock.get(&dict_name) {
+    if let Some(_) = dict_col_lock.get(&dict_name) {
         Ok(cx.string(format!(
             "Failed: dictionary {} exists, please use another name.",
             dict_name
@@ -49,10 +47,6 @@ fn segment(mut cx: FunctionContext) -> JsResult<JsArray> {
 
 #[neon::main]
 fn main(mut cx: ModuleContext) -> NeonResult<()> {
-    {
-        let mut dict_collect = DICT_COLLECTION.lock().unwrap();
-        dict_collect.insert("default".to_string(), Box::from(Newmm::new(None)));
-    }
     cx.export_function("loadDict", load_dict)?;
     cx.export_function("segment", segment)?;
     Ok(())
diff --git a/nlpo3-python/src/lib.rs b/nlpo3-python/src/lib.rs
index f225b7f..0fb55bc 100644
--- a/nlpo3-python/src/lib.rs
+++ b/nlpo3-python/src/lib.rs
@@ -49,9 +49,7 @@ fn segment(
 #[pyfunction]
 fn load_dict(file_path: &str, dict_name: &str) -> PyResult<String> {
     let mut dict_col_lock = DICT_COLLECTION.lock().unwrap();
-    if dict_name == "default" {
-        Ok(format!("Failed: 'default' dictionary name is reserved"))
-    } else if let Some(_) = dict_col_lock.get(dict_name) {
+    if let Some(_) = dict_col_lock.get(dict_name) {
         Ok(format!(
             "Failed: dictionary {} exists, please use another name.",
             dict_name
@@ -69,10 +67,6 @@ fn load_dict(file_path: &str, dict_name: &str) -> PyResult<String> {
 
 #[pymodule]
 fn _nlpo3_python_backend(_py: Python, m: &PyModule) -> PyResult<()> {
-    {
-        let mut dict_collect = DICT_COLLECTION.lock().unwrap();
-        dict_collect.insert("default".to_string(), Box::from(Newmm::new(None)));
-    }
     m.add_function(wrap_pyfunction!(load_dict, m)?)?;
     m.add_function(wrap_pyfunction!(segment, m)?)?;
     Ok(())
diff --git a/src/tokenizer/dict_reader_custom.rs b/src/tokenizer/dict_reader_custom.rs
index b0e630b..8ceed7a 100644
--- a/src/tokenizer/dict_reader_custom.rs
+++ b/src/tokenizer/dict_reader_custom.rs
@@ -5,22 +5,11 @@ use std::io::BufReader;
 use std::{error::Error, io::prelude::*};
 use std::{fs::File, path::PathBuf};
 
-const DEFAULT_DICT_FILE: &str = include_str!("../../words_th.txt");
-
 pub enum DictSource {
     WordList(Vec<String>),
     FilePath(PathBuf),
 }
 
-pub fn create_default_dict() -> Trie {
-    let default_dict = DEFAULT_DICT_FILE
-        .par_lines()
-        .map(|word| CustomString::new(word))
-        .filter(|word| !word.is_empty())
-        .collect::<Vec<CustomString>>();
-    Trie::new(&default_dict)
-}
-
 pub fn create_dict_trie(source: DictSource) -> Result<Trie, Box<dyn Error>> {
     match source {
         DictSource::FilePath(single_source) => {
diff --git a/src/tokenizer/newmm_custom.rs b/src/tokenizer/newmm_custom.rs
index b66a78a..b06503a 100644
--- a/src/tokenizer/newmm_custom.rs
+++ b/src/tokenizer/newmm_custom.rs
@@ -1,6 +1,6 @@
 use super::super::fixed_bytes_str::four_bytes::{CustomStringBytesSlice, CustomStringBytesVec};
 use super::{
-    dict_reader_custom::{create_default_dict, create_dict_trie, DictSource},
+    dict_reader_custom::{create_dict_trie, DictSource},
     tcc_custom,
     tokenizer_trait::Tokenizer,
     trie_custom::Trie,
@@ -98,16 +98,11 @@ impl Display for BFSSearchError {
 impl Error for BFSSearchError {}
 
 impl Newmm {
-    pub fn new(dict_path: Option<&str>) -> Self {
-        match dict_path {
-            None => Self {
-                dict: Box::from(create_default_dict()),
-            },
-            Some(path) => Self {
-                dict: Box::from(
-                    create_dict_trie(DictSource::FilePath(PathBuf::from(path))).unwrap(),
-                ),
-            },
+    pub fn new(dict_path: &str) -> Self {
+        Self {
+            dict: Box::from(
+                create_dict_trie(DictSource::FilePath(PathBuf::from(dict_path))).unwrap(),
+            ),
         }
     }
 


Test patch:
diff --git a/tests/test_tokenizer.rs b/tests/test_tokenizer.rs
index 2faa6f7..3e6e686 100644
--- a/tests/test_tokenizer.rs
+++ b/tests/test_tokenizer.rs
@@ -3,8 +3,11 @@ use nlpo3::tokenizer::tokenizer_trait::Tokenizer;
 const FIRST_TEXT: &str = "นิสสันผ่อนจนเพลียนาวาร่า..";
 const SECOND_TEXT: &str =
     "อาชญากรรมทางการแพทย์.. หลอกลวงคนไข้ผ่าตัด ตัดหมอนรองข้อเข่าอำพราง รพ.กรุงเทพภูเก็ตปลอมเวชระเบียน ตอนที่๑.";
+const DEFAULT_DICT_PATH: &str =  "/words_th.txt"; // relative to cargo
 #[test]
 fn test_long_text_byte_tokenizer() {
+    let mut relative_test_dict_path = env!("CARGO_MANIFEST_DIR").to_string();
+    relative_test_dict_path.push_str(DEFAULT_DICT_PATH);
     let long_text = [
         "ไต้หวัน (แป่ะเอ๋ยี้: Tâi-oân; ไต่อวัน) หรือ ไถวาน ",
         "(อักษรโรมัน: Taiwan; จีนตัวย่อ: 台湾; จีนตัวเต็ม: 臺灣/台灣; พินอิน: ",
@@ -148,17 +151,21 @@ fn test_long_text_byte_tokenizer() {
         " ทำให้เป็นประเทศแรกในทวีปเอเชียที่ผ่านกฎหมายดังกล่าว[18][19]",
     ]
     .join("");
-
-    let newmm_default_dict = NewmmCustom::new(None);
-    let result = newmm_default_dict.segment(&long_text, false, true).unwrap();
-
-    let safe_result = newmm_default_dict.segment(&long_text, true, true).unwrap();
+    let newmm_default_dict = NewmmCustom::new(&relative_test_dict_path);
+    let result = newmm_default_dict
+        .segment(&long_text, false, true)
+        .unwrap();
+    let safe_result = newmm_default_dict
+        .segment(&long_text, true, true)
+        .unwrap();
     assert_eq!(result.len(), 1889);
     assert_eq!(safe_result.len(), 1991);
 }
 #[test]
 fn test_standard_short_word() {
-    let newmm_default_dict = NewmmCustom::new(None);
+    let mut relative_test_dict_path = env!("CARGO_MANIFEST_DIR").to_string();
+    relative_test_dict_path.push_str(DEFAULT_DICT_PATH);
+    let newmm_default_dict = NewmmCustom::new(&relative_test_dict_path);
     assert_eq!(
         newmm_default_dict.segment_to_string("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", false, false),
         ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"]
@@ -186,7 +193,9 @@ fn test_standard_short_word() {
 }
 #[test]
 fn test_with_some_real_data() {
-    let newmm_default_dict = NewmmCustom::new(None);
+    let mut relative_test_dict_path = env!("CARGO_MANIFEST_DIR").to_string();
+    relative_test_dict_path.push_str(DEFAULT_DICT_PATH);
+    let newmm_default_dict = NewmmCustom::new(&relative_test_dict_path);
     assert_eq!(
         newmm_default_dict.segment_to_string(FIRST_TEXT, false, false),
         ["นิสสัน", "ผ่อน", "จน", "เพลีย", "นาวา", "ร่า", ".."]
@@ -224,7 +233,9 @@ fn test_with_some_real_data() {
 }
 #[test]
 fn test_thai_number() {
-    let newmm_default_dict = NewmmCustom::new(None);
+    let mut relative_test_dict_path = env!("CARGO_MANIFEST_DIR").to_string();
+    relative_test_dict_path.push_str(DEFAULT_DICT_PATH);
+    let newmm_default_dict = NewmmCustom::new(&relative_test_dict_path);
     assert_eq!(
         newmm_default_dict.segment_to_string("๑๙...", false, false),
         ["๑๙", "..."]