Repository: PyThaiNLP/pythainlp

Issue:
Thai character splitter to display cell
## Detailed description

This feature will split Thai text to display cell. E.g. แม่น้ำอยู่ที่ไหน -> แ ม่ น้ํ า อ ยู่ ที่ ไ ห น.

It will split Thai text to character but It doesn't split tone marks.

Discussion:
@bact: In Solaris Common Desktop Environment, we call this a "display cell" https://docs.oracle.com/cd/E19455-01/806-1360/6jalch36v/index.html@wannaphong: > In Solaris Common Desktop Environment, we call this a "display cell" https://docs.oracle.com/cd/E19455-01/806-1360/6jalch36v/index.html

Thank you!

Solution patch:
diff --git a/docs/api/tokenize.rst b/docs/api/tokenize.rst
index 41952d74..5fe02fdc 100644
--- a/docs/api/tokenize.rst
+++ b/docs/api/tokenize.rst
@@ -44,6 +44,8 @@ Modules
     
     The `Tokenizer` class is a versatile tool for customizing tokenization processes and managing tokenization models. It provides various methods and attributes to fine-tune tokenization according to your specific needs.
 
+.. autoclass:: display_cell_tokenize
+
 Tokenization Engines
 --------------------
 
diff --git a/pythainlp/corpus/words_th.txt b/pythainlp/corpus/words_th.txt
index 5d9ec377..e5c825b1 100644
--- a/pythainlp/corpus/words_th.txt
+++ b/pythainlp/corpus/words_th.txt
@@ -3685,6 +3685,7 @@
 การนิพนธ์
 การนิยาม
 การ์นิเยร์
+การ์ฟิลด์
 การแนะแนว
 การโน้ม
 การโน้มน้าว
@@ -11716,6 +11717,7 @@
 โคเนื้อ
 โค่น
 โคน
+โคนัน
 โคบอลต์
 โคบาล
 โคบุตร
@@ -39129,6 +39131,7 @@
 มินิมาร์ท
 มินิ
 มินิสเกิร์ต
+มินเนี่ยน
 มิเนโซต้า
 มิบังควร
 มิ้มดำ
diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py
index 40a83d36..e9727a35 100644
--- a/pythainlp/tokenize/__init__.py
+++ b/pythainlp/tokenize/__init__.py
@@ -16,6 +16,7 @@ __all__ = [
     "syllable_tokenize",
     "word_detokenize",
     "word_tokenize",
+    "display_cell_tokenize",
 ]
 
 from pythainlp.corpus import thai_syllables, thai_words
@@ -38,6 +39,7 @@ from pythainlp.tokenize.core import (
     syllable_tokenize,
     word_detokenize,
     word_tokenize,
+    display_cell_tokenize,
 )
 
 from pythainlp.corpus import get_corpus as _get_corpus
diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py
index 64cdddde..d6826fe0 100644
--- a/pythainlp/tokenize/attacut.py
+++ b/pythainlp/tokenize/attacut.py
@@ -8,7 +8,7 @@ Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai
 :See Also:
     * `GitHub repository <https://github.com/PyThaiNLP/attacut>`_
 """
-from typing import List
+from typing import Dict, List
 
 from attacut import Tokenizer
 
@@ -26,6 +26,9 @@ class AttacutTokenizer:
         return self._tokenizer.tokenize(text)
 
 
+_tokenizers: Dict[str, AttacutTokenizer] = {}
+
+
 def segment(text: str, model: str = "attacut-sc") -> List[str]:
     """
     Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai
@@ -40,6 +43,8 @@ def segment(text: str, model: str = "attacut-sc") -> List[str]:
     if not text or not isinstance(text, str):
         return []
 
-    _tokenizer = AttacutTokenizer(model)
+    global _tokenizers
+    if model not in _tokenizers:
+        _tokenizers[model] = AttacutTokenizer(model)
 
-    return _tokenizer.tokenize(text)
+    return _tokenizers[model].tokenize(text)
diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py
index 1f8304c4..c3e73649 100644
--- a/pythainlp/tokenize/core.py
+++ b/pythainlp/tokenize/core.py
@@ -733,6 +733,46 @@ def syllable_tokenize(
     )
 
 
+def display_cell_tokenize(text: str) -> List[str]:
+    """
+    Display cell tokenizer.
+
+    Tokenizes Thai text into display cells without splitting tone marks.
+
+    :param str text: text to be tokenized
+    :return: list of display cells
+    :rtype: List[str]
+    :Example:
+
+    Tokenize Thai text into display cells::
+
+        from pythainlp.tokenize import display_cell_tokenize
+
+        text = "แม่น้ำอยู่ที่ไหน"
+        display_cell_tokenize(text)
+        # output: ['แ', 'ม่', 'น้ํ', 'า', 'อ', 'ยู่', 'ที่', 'ไ', 'ห', 'น']
+    """
+    if not text or not isinstance(text, str):
+        return []
+
+    display_cells = []
+    current_cell = ""
+    text = text.replace("ำ", "ํา")
+
+    for char in text:
+        if re.match(r"[\u0E31\u0E34-\u0E3A\u0E47-\u0E4E]", char):
+            current_cell += char
+        else:
+            if current_cell:
+                display_cells.append(current_cell)
+            current_cell = char
+
+    if current_cell:
+        display_cells.append(current_cell)
+
+    return display_cells
+
+
 class Tokenizer:
     """
     Tokenizer class for a custom tokenizer.
diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py
index 8bd21bf4..42b9e5bd 100644
--- a/pythainlp/tokenize/longest.py
+++ b/pythainlp/tokenize/longest.py
@@ -12,7 +12,7 @@ on the codes from Patorn Utenpattanun.
 
 """
 import re
-from typing import List, Union
+from typing import Dict, List, Union
 
 from pythainlp import thai_tonemarks
 from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE
@@ -134,16 +134,25 @@ class LongestMatchTokenizer:
                     token_statuses.append(_KNOWN)
                 begin_pos += len(match)
 
-        return tokens
+        # Group consecutive spaces into one token
+        grouped_tokens = []
+        for token in tokens:
+            if token.isspace() and grouped_tokens and grouped_tokens[-1].isspace():
+                grouped_tokens[-1] += token
+            else:
+                grouped_tokens.append(token)
+
+        return grouped_tokens
 
     def tokenize(self, text: str) -> List[str]:
         tokens = self.__segment(text)
         return tokens
 
 
-def segment(
-    text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE
-) -> List[str]:
+_tokenizers: Dict[int, LongestMatchTokenizer] = {}
+
+
+def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]:
     """
     Dictionary-based longest matching word segmentation.
 
@@ -157,4 +166,9 @@ def segment(
     if not custom_dict:
         custom_dict = DEFAULT_WORD_DICT_TRIE
 
-    return LongestMatchTokenizer(custom_dict).tokenize(text)
+    global _tokenizers
+    custom_dict_ref_id = id(custom_dict)
+    if custom_dict_ref_id not in _tokenizers:
+        _tokenizers[custom_dict_ref_id] = LongestMatchTokenizer(custom_dict)
+
+    return _tokenizers[custom_dict_ref_id].tokenize(text)
diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py
index cd9f238c..c999fb7b 100644
--- a/pythainlp/tokenize/newmm.py
+++ b/pythainlp/tokenize/newmm.py
@@ -182,7 +182,7 @@ def segment(
         # try to break by space first
         space_idx = sample.rfind(" ")
         if space_idx >= 0:
-            cut_pos = space_idx + 1
+            cut_pos = space_idx + 1 + _TEXT_SCAN_BEGIN
         else:
             tokens = list(_onecut(sample, custom_dict))
             token_max_idx = 0
diff --git a/pythainlp/tokenize/pyicu.py b/pythainlp/tokenize/pyicu.py
index 492417f0..213080a8 100644
--- a/pythainlp/tokenize/pyicu.py
+++ b/pythainlp/tokenize/pyicu.py
@@ -15,9 +15,10 @@ from typing import List
 
 from icu import BreakIterator, Locale
 
+bd = BreakIterator.createWordInstance(Locale("th"))
 
 def _gen_words(text: str) -> str:
-    bd = BreakIterator.createWordInstance(Locale("th"))
+    global bd
     bd.setText(text)
     p = bd.first()
     for q in bd:


Test patch:
diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py
index 054a2ab7..e7ee674f 100644
--- a/tests/core/test_tokenize.py
+++ b/tests/core/test_tokenize.py
@@ -19,6 +19,7 @@ from pythainlp.tokenize import (
     tcc_p,
     word_detokenize,
     word_tokenize,
+    display_cell_tokenize,
 )
 from pythainlp.util import dict_trie
 
@@ -390,6 +391,40 @@ class TokenizeTestCase(unittest.TestCase):
             longest_tokenizer.word_tokenize("เฉียบพลัน"),
             ["เฉียบพลัน"],
         )
+        self.assertEqual(
+            longest.segment("ทดสอบ  ทดสอบ  ทดสอบ"),
+            ["ทดสอบ", "  ", "ทดสอบ", "  ", "ทดสอบ"],
+        )
+        self.assertEqual(
+            longest.segment("ทดสอบ  ทดสอบ"),
+            ["ทดสอบ", "  ", "ทดสอบ"],
+        )
+        self.assertEqual(
+            longest.segment("ทดสอบ    ทดสอบ"),
+            ["ทดสอบ", "    ", "ทดสอบ"],
+        )
+
+    def test_longest_custom_dict(self):
+        """Test switching the custom dict on longest segment function"""
+
+        self.assertEqual(
+            word_tokenize("ทดสอบ  ทดสอบ", engine="longest"),
+            ["ทดสอบ", "  ", "ทดสอบ"],
+        )
+        self.assertEqual(
+            word_tokenize(
+                "ปวดเฉียบพลัน", engine="longest", custom_dict=dict_trie(["ปวดเฉียบพลัน"])
+            ),
+            ["ปวดเฉียบพลัน"],
+        )
+        self.assertEqual(
+            word_tokenize("ทดสอบทดสอบ", engine="longest", custom_dict=dict_trie(["ทดสอบท"])),
+            ["ทดสอบท", "ดสอบ"],
+        )
+        self.assertEqual(
+            word_tokenize("ทดสอบ  ทดสอบ", engine="longest"),
+            ["ทดสอบ", "  ", "ทดสอบ"],
+        )
 
     def test_mm(self):
         self.assertEqual(multi_cut.segment(None), [])
@@ -604,3 +639,13 @@ class TokenizeTestCase(unittest.TestCase):
         # )
         self.assertEqual(list(tcc_p.tcc("")), [])
         self.assertEqual(tcc_p.tcc_pos(""), set())
+
+    def test_display_cell_tokenize(self):
+        self.assertEqual(display_cell_tokenize(""), [])
+        self.assertEqual(
+            display_cell_tokenize("แม่น้ำอยู่ที่ไหน"),
+            ["แ", "ม่", "น้ํ", "า", "อ", "ยู่", "ที่", "ไ", "ห", "น"]
+        )
+        self.assertEqual(display_cell_tokenize("สวัสดี"), ['ส', 'วั', 'ส', 'ดี'])
+        self.assertEqual(display_cell_tokenize("ทดสอบ"), ["ท", "ด", "ส", "อ", "บ"])
+        self.assertEqual(display_cell_tokenize("ภาษาไทย"), ["ภ", "า", "ษ", "า", "ไ", "ท", "ย"])