Repository: PyThaiNLP/nlpo3 (language: python)

Issue:
Add from_word_list() for Newmm tokenizer
Trying to make it is possible to create a NewmmTokenizer from Vec<String>.

Very new to Rust. Please check if this is the way people doing a proper Rust.
- Maybe it's less computational if we can just use a fixed array of &str?
- Does it possible to merge this with the existing `new()` ?

The use case is the user may already have a word list (or a trie) that they would like to fine tune
(removing/adding words) and create different tokenizers from this evolving list of words.
For easy processing, the list of word is in the memory not in the file.

Update non-Thai regex and Thai dict to follow PyThaiNLP changes
- Update non-Thai regex
- Update standard dict
- Fix gitignore "target" path


Discussion / hints:
Also not sure how to properly design this, with bindings in mind.

Python user will likely to come up with List[str] or any Iterable[str].


Fix patch:
diff --git a/nlpo3-python/src/lib.rs b/nlpo3-python/src/lib.rs
index 9bbff98..d1f313f 100644
--- a/nlpo3-python/src/lib.rs
+++ b/nlpo3-python/src/lib.rs
@@ -17,7 +17,8 @@ use pyo3::types::PyString;
 use pyo3::{exceptions, wrap_pyfunction};
 
 lazy_static! {
-    static ref DICT_COLLECTION: Mutex<HashMap<String, Box<NewmmTokenizer>>> = Mutex::new(HashMap::new());
+    static ref DICT_COLLECTION: Mutex<HashMap<String, Box<NewmmTokenizer>>> =
+        Mutex::new(HashMap::new());
 }
 
 /// Break text into tokens.
@@ -67,6 +68,42 @@ fn load_dict(file_path: &str, dict_name: &str) -> PyResult<(String, bool)> {
     }
 }
 
+/// Add words to existing dictionary
+#[pyfunction]
+fn add_word(dict_name: &str, words: Vec<&str>) -> PyResult<(String, bool)> {
+    let mut dict_col_lock = DICT_COLLECTION.lock().unwrap();
+    if let Some(newmm_dict) = dict_col_lock.get(dict_name) {
+        newmm_dict.add_word(&words);
+        Ok((format!("Add new word(s) successfully."), true))
+    } else {
+        Ok((
+            format!(
+                "Cannot add new word(s) - dictionary instance named '{}' does not exist.",
+                dict_name
+            ),
+            false,
+        ))
+    }
+}
+
+/// Remove words from existing dictionary
+#[pyfunction]
+fn remove_word(dict_name: &str, words: Vec<&str>) -> PyResult<(String, bool)> {
+    let mut dict_col_lock = DICT_COLLECTION.lock().unwrap();
+    if let Some(newmm_dict) = dict_col_lock.get(dict_name) {
+        newmm_dict.remove_word(&words);
+        Ok((format!("Remove word(s) successfully."), true))
+    } else {
+        Ok((
+            format!(
+                "Cannot remove word(s) - dictionary instance named '{}' does not exist.",
+                dict_name
+            ),
+            false,
+        ))
+    }
+}
+
 #[pymodule]
 fn _nlpo3_python_backend(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(load_dict, m)?)?;
diff --git a/src/tokenizer/newmm.rs b/src/tokenizer/newmm.rs
index 0e6b505..535d441 100644
--- a/src/tokenizer/newmm.rs
+++ b/src/tokenizer/newmm.rs
@@ -116,9 +116,19 @@ impl NewmmTokenizer {
 
     pub fn from_word_list(word_list: Vec<String>) -> Self {
         NewmmTokenizer {
-            dict: Box::from(
-                create_dict_trie(DictSource::WordList(word_list)).unwrap(),
-            ),
+            dict: Box::from(create_dict_trie(DictSource::WordList(word_list)).unwrap()),
+        }
+    }
+
+    pub fn add_word(&mut self, word_list: &[&str]) {
+        for word in word_list {
+            self.dict.add(&CustomString::new(word));
+        }
+    }
+
+    pub fn remove_word(&mut self, word_list: &[&str]) {
+        for word in word_list {
+            self.dict.remove(&CustomString::new(word));
         }
     }
 
diff --git a/src/tokenizer/trie_char.rs b/src/tokenizer/trie_char.rs
index 275ce8d..2fab39f 100644
--- a/src/tokenizer/trie_char.rs
+++ b/src/tokenizer/trie_char.rs
@@ -67,16 +67,18 @@ impl TrieNode {
     }
 
     fn remove_word(&mut self, input_word: &CustomString) {
-        let word = input_word;
+        let mut word = input_word;
         let char_count = word.chars_len();
         // if has at least 1 char
-        if char_count >= BYTES_PER_CHAR {
+        if char_count >= 1 {
             let character = word.get_chars_content().get(0).unwrap();
             if let Some(child) = self.find_mut_child(character) {
-                // word = &word.substring(1, word.chars_len());
+                // move 1 character
+                let substring_of_word = word.substring(1, word.chars_len());
                 if char_count == 1 {
                     child.set_not_end();
                 }
+                word = &substring_of_word;
                 child.remove_word(word);
                 if !child.end && child.children.is_empty() {
                     self.remove_child(character);


Test patch:
diff --git a/tests/test_tokenizer.rs b/tests/test_tokenizer.rs
index 31d5262..6c8259f 100644
--- a/tests/test_tokenizer.rs
+++ b/tests/test_tokenizer.rs
@@ -211,6 +211,25 @@ fn test_standard_short_word() {
     );
 }
 
+#[test]
+fn test_with_add_or_remove_word() {
+    let mut relative_test_dict_path = env!("CARGO_MANIFEST_DIR").to_string();
+    relative_test_dict_path.push_str(DEFAULT_DICT_PATH);
+
+    let mut tokenizer = NewmmTokenizer::new(&relative_test_dict_path);
+    tokenizer.add_word(&["ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"]); // exaggerated word
+    assert_eq!(
+        tokenizer.segment_to_string("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", false, false),
+        ["ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"]
+    );
+    tokenizer.remove_word(&["ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"]); // exaggerated word
+    tokenizer.remove_word(&["ภาษาไทย"]); // exaggerated word
+    assert_eq!(
+        tokenizer.segment_to_string("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", false, false),
+        ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"]
+    );
+}
+
 #[test]
 fn test_with_some_real_data() {
     let mut relative_test_dict_path = env!("CARGO_MANIFEST_DIR").to_string();