Repository: PyThaiNLP/pythainlp (python)

Issue:
Thai character splitter to display cell
## Detailed description

This feature will split Thai text to display cell. E.g. แม่น้ำอยู่ที่ไหน -> แ ม่ น้ํ า อ ยู่ ที่ ไ ห น.

It will split Thai text to character but It doesn't split tone marks.

Pull request description:
Add display cell tokenizer
Fixes #663

Add a new function `display_cell_tokenize` to split Thai text into display cells without splitting tone marks.

* **New Functionality**
  - Add `display_cell_tokenize` function in `pythainlp/tokenize/core.py` to handle the splitting of Thai text into display cells.
  - Ensure the function does not split tone marks.
* **Initialization**
  - Update `pythainlp/tokenize/__init__.py` to include the new `display_cell_tokenize` function in the `__all__` list.
* **Testing**
  - Add tests for the `display_cell_tokenize` function in `tests/core/test_tokenize.py`.

---

For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/PyThaiNLP/pythainlp/pull/1058?shareId=7868b390-9242-49d0-a7d3-6a1f13b16ade).

Patch:
diff --git a/docs/api/tokenize.rst b/docs/api/tokenize.rst
index 41952d74..5fe02fdc 100644
--- a/docs/api/tokenize.rst
+++ b/docs/api/tokenize.rst
@@ -44,6 +44,8 @@ Modules
     
     The `Tokenizer` class is a versatile tool for customizing tokenization processes and managing tokenization models. It provides various methods and attributes to fine-tune tokenization according to your specific needs.
 
+.. autoclass:: display_cell_tokenize
+
 Tokenization Engines
 --------------------
 
diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py
index 40a83d36..e9727a35 100644
--- a/pythainlp/tokenize/__init__.py
+++ b/pythainlp/tokenize/__init__.py
@@ -16,6 +16,7 @@ __all__ = [
     "syllable_tokenize",
     "word_detokenize",
     "word_tokenize",
+    "display_cell_tokenize",
 ]
 
 from pythainlp.corpus import thai_syllables, thai_words
@@ -38,6 +39,7 @@ from pythainlp.tokenize.core import (
     syllable_tokenize,
     word_detokenize,
     word_tokenize,
+    display_cell_tokenize,
 )
 
 from pythainlp.corpus import get_corpus as _get_corpus
diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py
index 1f8304c4..c3e73649 100644
--- a/pythainlp/tokenize/core.py
+++ b/pythainlp/tokenize/core.py
@@ -733,6 +733,46 @@ def syllable_tokenize(
     )
 
 
+def display_cell_tokenize(text: str) -> List[str]:
+    """
+    Display cell tokenizer.
+
+    Tokenizes Thai text into display cells without splitting tone marks.
+
+    :param str text: text to be tokenized
+    :return: list of display cells
+    :rtype: List[str]
+    :Example:
+
+    Tokenize Thai text into display cells::
+
+        from pythainlp.tokenize import display_cell_tokenize
+
+        text = "แม่น้ำอยู่ที่ไหน"
+        display_cell_tokenize(text)
+        # output: ['แ', 'ม่', 'น้ํ', 'า', 'อ', 'ยู่', 'ที่', 'ไ', 'ห', 'น']
+    """
+    if not text or not isinstance(text, str):
+        return []
+
+    display_cells = []
+    current_cell = ""
+    text = text.replace("ำ", "ํา")
+
+    for char in text:
+        if re.match(r"[\u0E31\u0E34-\u0E3A\u0E47-\u0E4E]", char):
+            current_cell += char
+        else:
+            if current_cell:
+                display_cells.append(current_cell)
+            current_cell = char
+
+    if current_cell:
+        display_cells.append(current_cell)
+
+    return display_cells
+
+
 class Tokenizer:
     """
     Tokenizer class for a custom tokenizer.


Test patch:
diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py
index 054a2ab7..f752a1b9 100644
--- a/tests/core/test_tokenize.py
+++ b/tests/core/test_tokenize.py
@@ -19,6 +19,7 @@ from pythainlp.tokenize import (
     tcc_p,
     word_detokenize,
     word_tokenize,
+    display_cell_tokenize,
 )
 from pythainlp.util import dict_trie
 
@@ -604,3 +605,13 @@ class TokenizeTestCase(unittest.TestCase):
         # )
         self.assertEqual(list(tcc_p.tcc("")), [])
         self.assertEqual(tcc_p.tcc_pos(""), set())
+
+    def test_display_cell_tokenize(self):
+        self.assertEqual(display_cell_tokenize(""), [])
+        self.assertEqual(
+            display_cell_tokenize("แม่น้ำอยู่ที่ไหน"),
+            ["แ", "ม่", "น้ํ", "า", "อ", "ยู่", "ที่", "ไ", "ห", "น"]
+        )
+        self.assertEqual(display_cell_tokenize("สวัสดี"), ['ส', 'วั', 'ส', 'ดี'])
+        self.assertEqual(display_cell_tokenize("ทดสอบ"), ["ท", "ด", "ส", "อ", "บ"])
+        self.assertEqual(display_cell_tokenize("ภาษาไทย"), ["ภ", "า", "ษ", "า", "ไ", "ท", "ย"])