Repository: PyThaiNLP/pythainlp

Issue:
bug: Should raise error for using custom dict with unsupported word tokenization engine
### Description

Should raise an error when `custom_dict` is passed into `word_tokenizer` function but the selected engine is not supported custom_dict

### Expected results

Should either:
1. Raise `NotImplementedError` if the user passed in custom_dict to unsupported tokenizer engines
or
2. Raise a warning

### Current results

The current code completely ignores custom dict behavior when its engine not supported

https://github.com/PyThaiNLP/pythainlp/blob/3aa57c6bc62bdb91b6e1713303fcfa7e15855b1b/pythainlp/tokenize/core.py#L228-L235

### Steps to reproduce

```python
from pythainlp.tokenize import word_tokenize
from pythainlp.util import dict_trie


custom_dict = dict_trie(set(['tsts']))
word_tokenize('tstshel', custom_dict=custom_dict, engine = 'attacut')
```

### PyThaiNLP version

Commit hash 3aa57c6

### Python version

all

### Operating system and version

all

### More info

_No response_

### Possible solution

_No response_

### Files

_No response_

Discussion:

Solution patch:
diff --git a/docker_requirements.txt b/docker_requirements.txt
index 45fdf336..18ed5e56 100644
--- a/docker_requirements.txt
+++ b/docker_requirements.txt
@@ -18,7 +18,7 @@ OSKut==1.3
 pandas==2.2.*
 panphon==0.21.2
 phunspell==0.1.6
-protobuf==5.29.2
+protobuf==5.29.3
 pyicu==2.14
 python-crfsuite==0.9.11
 requests==2.32.*
@@ -35,7 +35,7 @@ tensorflow==2.18.0
 thai-nner==0.3
 tltk==1.6.8
 torch==1.13.1
-transformers==4.47.1
+transformers==4.48.0
 ufal.chu-liu-edmonds==1.0.3
 wtpsplit==1.3.0
 wunsen==0.0.3
diff --git a/docs/api/khavee.rst b/docs/api/khavee.rst
index 97e7117e..ea6dcf13 100644
--- a/docs/api/khavee.rst
+++ b/docs/api/khavee.rst
@@ -49,6 +49,7 @@ Example
 Here's a basic example of how to use the `KhaveeVerifier` class to verify Thai poetry:
 
 ::
+
   from pythainlp.khavee import KhaveeVerifier
   
   # Initialize a KhaveeVerifier instance
diff --git a/docs/api/tokenize.rst b/docs/api/tokenize.rst
index 41952d74..5fe02fdc 100644
--- a/docs/api/tokenize.rst
+++ b/docs/api/tokenize.rst
@@ -44,6 +44,8 @@ Modules
     
     The `Tokenizer` class is a versatile tool for customizing tokenization processes and managing tokenization models. It provides various methods and attributes to fine-tune tokenization according to your specific needs.
 
+.. autoclass:: display_cell_tokenize
+
 Tokenization Engines
 --------------------
 
diff --git a/docs/api/util.rst b/docs/api/util.rst
index 76452e12..1979e44e 100644
--- a/docs/api/util.rst
+++ b/docs/api/util.rst
@@ -283,6 +283,11 @@ Modules
 
     The `Trie` class is a data structure for efficient dictionary operations. It's a valuable resource for managing and searching word lists and dictionaries in a structured and efficient manner.
 
+.. autofunction:: longest_common_subsequence
+    :noindex:
+
+    The `longest_common_subsequence` function is find the longest common subsequence between two strings.
+
 .. autofunction:: pythainlp.util.morse.morse_encode
     :noindex:
 
diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py
index 40a83d36..e9727a35 100644
--- a/pythainlp/tokenize/__init__.py
+++ b/pythainlp/tokenize/__init__.py
@@ -16,6 +16,7 @@ __all__ = [
     "syllable_tokenize",
     "word_detokenize",
     "word_tokenize",
+    "display_cell_tokenize",
 ]
 
 from pythainlp.corpus import thai_syllables, thai_words
@@ -38,6 +39,7 @@ from pythainlp.tokenize.core import (
     syllable_tokenize,
     word_detokenize,
     word_tokenize,
+    display_cell_tokenize,
 )
 
 from pythainlp.corpus import get_corpus as _get_corpus
diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py
index 64cdddde..d6826fe0 100644
--- a/pythainlp/tokenize/attacut.py
+++ b/pythainlp/tokenize/attacut.py
@@ -8,7 +8,7 @@ Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai
 :See Also:
     * `GitHub repository <https://github.com/PyThaiNLP/attacut>`_
 """
-from typing import List
+from typing import Dict, List
 
 from attacut import Tokenizer
 
@@ -26,6 +26,9 @@ class AttacutTokenizer:
         return self._tokenizer.tokenize(text)
 
 
+_tokenizers: Dict[str, AttacutTokenizer] = {}
+
+
 def segment(text: str, model: str = "attacut-sc") -> List[str]:
     """
     Wrapper for AttaCut - Fast and Reasonably Accurate Word Tokenizer for Thai
@@ -40,6 +43,8 @@ def segment(text: str, model: str = "attacut-sc") -> List[str]:
     if not text or not isinstance(text, str):
         return []
 
-    _tokenizer = AttacutTokenizer(model)
+    global _tokenizers
+    if model not in _tokenizers:
+        _tokenizers[model] = AttacutTokenizer(model)
 
-    return _tokenizer.tokenize(text)
+    return _tokenizers[model].tokenize(text)
diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py
index 1f8304c4..bcc166c0 100644
--- a/pythainlp/tokenize/core.py
+++ b/pythainlp/tokenize/core.py
@@ -111,10 +111,10 @@ def word_tokenize(
     :param str engine: name of the tokenizer to be used
     :param pythainlp.util.Trie custom_dict: dictionary trie (some engine may not support)
     :param bool keep_whitespace: True to keep whitespace, a common mark
-                                 for end of phrase in Thai.
-                                 Otherwise, whitespace is omitted.
+                                  for end of phrase in Thai.
+                                  Otherwise, whitespace is omitted.
     :param bool join_broken_num: True to rejoin formatted numeric that could be wrongly separated.
-                                 Otherwise, formatted numeric could be wrongly separated.
+                                  Otherwise, formatted numeric could be wrongly separated.
 
     :return: list of words
     :rtype: List[str]
@@ -221,6 +221,18 @@ def word_tokenize(
 
     segments = []
 
+    if custom_dict and engine in (
+        "attacut",
+        "icu",
+        "nercut",
+        "sefr_cut",
+        "tltk",
+        "oskut"
+    ):
+        raise NotImplementedError(
+            f"The {engine} engine does not support custom dictionaries."
+        )
+
     if engine in ("newmm", "onecut"):
         from pythainlp.tokenize.newmm import segment
 
@@ -366,7 +378,7 @@ def sent_tokenize(
             and ``wtp-large`` to use ``wtp-canine-s-12l`` model.
         * *whitespace+newline* - split by whitespace and newline.
         * *whitespace* - split by whitespace, specifically with \
-                         :class:`regex` pattern  ``r" +"``
+                          :class:`regex` pattern  ``r" +"``
     :Example:
 
     Split the text based on *whitespace*::
@@ -733,6 +745,46 @@ def syllable_tokenize(
     )
 
 
+def display_cell_tokenize(text: str) -> List[str]:
+    """
+    Display cell tokenizer.
+
+    Tokenizes Thai text into display cells without splitting tone marks.
+
+    :param str text: text to be tokenized
+    :return: list of display cells
+    :rtype: List[str]
+    :Example:
+
+    Tokenize Thai text into display cells::
+
+        from pythainlp.tokenize import display_cell_tokenize
+
+        text = "แม่น้ำอยู่ที่ไหน"
+        display_cell_tokenize(text)
+        # output: ['แ', 'ม่', 'น้ํ', 'า', 'อ', 'ยู่', 'ที่', 'ไ', 'ห', 'น']
+    """
+    if not text or not isinstance(text, str):
+        return []
+
+    display_cells = []
+    current_cell = ""
+    text = text.replace("ำ", "ํา")
+
+    for char in text:
+        if re.match(r"[\u0E31\u0E34-\u0E3A\u0E47-\u0E4E]", char):
+            current_cell += char
+        else:
+            if current_cell:
+                display_cells.append(current_cell)
+            current_cell = char
+
+    if current_cell:
+        display_cells.append(current_cell)
+
+    return display_cells
+
+
 class Tokenizer:
     """
     Tokenizer class for a custom tokenizer.
@@ -814,9 +866,9 @@ class Tokenizer:
                     used to create a trie, or an instantiated
                     :class:`pythainlp.util.Trie` object.
         :param str engine: choose between different options of tokenizer engines
-                           (i.e.  *newmm*, *mm*, *longest*, *deepcut*)
+                            (i.e.  *newmm*, *mm*, *longest*, *deepcut*)
         :param bool keep_whitespace: True to keep whitespace, a common mark
-                                    for end of phrase in Thai
+                                     for end of phrase in Thai
         """
         self.__trie_dict = Trie([])
         if custom_dict:
diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py
index 053f9735..42b9e5bd 100644
--- a/pythainlp/tokenize/longest.py
+++ b/pythainlp/tokenize/longest.py
@@ -12,7 +12,7 @@ on the codes from Patorn Utenpattanun.
 
 """
 import re
-from typing import List, Union
+from typing import Dict, List, Union
 
 from pythainlp import thai_tonemarks
 from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE
@@ -149,9 +149,10 @@ class LongestMatchTokenizer:
         return tokens
 
 
-def segment(
-    text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE
-) -> List[str]:
+_tokenizers: Dict[int, LongestMatchTokenizer] = {}
+
+
+def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]:
     """
     Dictionary-based longest matching word segmentation.
 
@@ -165,4 +166,9 @@ def segment(
     if not custom_dict:
         custom_dict = DEFAULT_WORD_DICT_TRIE
 
-    return LongestMatchTokenizer(custom_dict).tokenize(text)
+    global _tokenizers
+    custom_dict_ref_id = id(custom_dict)
+    if custom_dict_ref_id not in _tokenizers:
+        _tokenizers[custom_dict_ref_id] = LongestMatchTokenizer(custom_dict)
+
+    return _tokenizers[custom_dict_ref_id].tokenize(text)
diff --git a/pythainlp/tokenize/pyicu.py b/pythainlp/tokenize/pyicu.py
index 492417f0..213080a8 100644
--- a/pythainlp/tokenize/pyicu.py
+++ b/pythainlp/tokenize/pyicu.py
@@ -15,9 +15,10 @@ from typing import List
 
 from icu import BreakIterator, Locale
 
+bd = BreakIterator.createWordInstance(Locale("th"))
 
 def _gen_words(text: str) -> str:
-    bd = BreakIterator.createWordInstance(Locale("th"))
+    global bd
     bd.setText(text)
     p = bd.first()
     for q in bd:
diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py
index 9939ac08..f101ffe3 100644
--- a/pythainlp/util/__init__.py
+++ b/pythainlp/util/__init__.py
@@ -1,4 +1,4 @@
-﻿# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2025 PyThaiNLP Project
 # SPDX-FileType: SOURCE
 # SPDX-License-Identifier: Apache-2.0
@@ -26,6 +26,7 @@ __all__ = [
     "is_native_thai",
     "isthai",
     "isthaichar",
+    "longest_common_subsequence",
     "nectec_to_ipa",
     "normalize",
     "now_reign_year",
@@ -92,6 +93,7 @@ from pythainlp.util.keyboard import (
     thai_to_eng,
 )
 from pythainlp.util.keywords import find_keyword, rank
+from pythainlp.util.lcs import longest_common_subsequence
 from pythainlp.util.normalize import (
     maiyamok,
     normalize,
diff --git a/pythainlp/util/lcs.py b/pythainlp/util/lcs.py
new file mode 100644
index 00000000..104f5674
--- /dev/null
+++ b/pythainlp/util/lcs.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: 2016-2025 PyThaiNLP Project
+# SPDX-FileType: SOURCE
+# SPDX-License-Identifier: Apache-2.0
+
+def longest_common_subsequence(str1: str, str2: str) -> str:
+    """
+    Find the longest common subsequence between two strings.
+
+    :param str str1: The first string.
+    :param str str2: The second string.
+    :return: The longest common subsequence.
+    :rtype: str
+
+    :Example:
+    ::
+
+        from pythainlp.util.lcs import longest_common_subsequence
+
+        print(longest_common_subsequence("ABCBDAB", "BDCAB"))
+        # output: "BDAB"
+    """
+    m = len(str1)
+    n = len(str2)
+
+    # Create a 2D array to store lengths of longest common subsequence.
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+
+    # Build the dp array from bottom up.
+    for i in range(m + 1):
+        for j in range(n + 1):
+            if i == 0 or j == 0:
+                dp[i][j] = 0
+            elif str1[i - 1] == str2[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1] + 1
+            else:
+                dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
+
+    # Following code is used to print LCS
+    index = dp[m][n]
+
+    # Create a character array to store the lcs string
+    lcs = [""] * (index + 1)
+    lcs[index] = ""
+
+    # Start from the right-most-bottom-most corner and
+    # one by one store characters in lcs[]
+    i = m
+    j = n
+    while i > 0 and j > 0:
+
+        # If current character in str1 and str2 are same, then
+        # current character is part of LCS
+        if str1[i - 1] == str2[j - 1]:
+            lcs[index - 1] = str1[i - 1]
+            i -= 1
+            j -= 1
+            index -= 1
+
+        # If not same, then find the larger of two and
+        # go in the direction of larger value
+        elif dp[i - 1][j] > dp[i][j - 1]:
+            i -= 1
+        else:
+            j -= 1
+
+    return "".join(lcs)


Test patch:
diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py
index c1fd7ba0..534ded7f 100644
--- a/tests/core/test_tokenize.py
+++ b/tests/core/test_tokenize.py
@@ -19,6 +19,7 @@ from pythainlp.tokenize import (
     tcc_p,
     word_detokenize,
     word_tokenize,
+    display_cell_tokenize,
 )
 from pythainlp.util import dict_trie
 
@@ -355,6 +356,11 @@ class TokenizeTestCase(unittest.TestCase):
             "ไฟ", word_tokenize("รถไฟฟ้า", custom_dict=dict_trie(["ไฟ"]))
         )
 
+        with self.assertRaises(NotImplementedError):
+            word_tokenize(
+                "รถไฟฟ้า", custom_dict=dict_trie(["ไฟ"]), engine="icu"
+            )
+
     def test_etcc(self):
         self.assertEqual(etcc.segment(None), [])
         self.assertEqual(etcc.segment(""), [])
@@ -403,6 +409,28 @@ class TokenizeTestCase(unittest.TestCase):
             ["ทดสอบ", "    ", "ทดสอบ"],
         )
 
+    def test_longest_custom_dict(self):
+        """Test switching the custom dict on longest segment function"""
+
+        self.assertEqual(
+            word_tokenize("ทดสอบ  ทดสอบ", engine="longest"),
+            ["ทดสอบ", "  ", "ทดสอบ"],
+        )
+        self.assertEqual(
+            word_tokenize(
+                "ปวดเฉียบพลัน", engine="longest", custom_dict=dict_trie(["ปวดเฉียบพลัน"])
+            ),
+            ["ปวดเฉียบพลัน"],
+        )
+        self.assertEqual(
+            word_tokenize("ทดสอบทดสอบ", engine="longest", custom_dict=dict_trie(["ทดสอบท"])),
+            ["ทดสอบท", "ดสอบ"],
+        )
+        self.assertEqual(
+            word_tokenize("ทดสอบ  ทดสอบ", engine="longest"),
+            ["ทดสอบ", "  ", "ทดสอบ"],
+        )
+
     def test_mm(self):
         self.assertEqual(multi_cut.segment(None), [])
         self.assertEqual(multi_cut.segment(""), [])
@@ -616,3 +644,13 @@ class TokenizeTestCase(unittest.TestCase):
         # )
         self.assertEqual(list(tcc_p.tcc("")), [])
         self.assertEqual(tcc_p.tcc_pos(""), set())
+
+    def test_display_cell_tokenize(self):
+        self.assertEqual(display_cell_tokenize(""), [])
+        self.assertEqual(
+            display_cell_tokenize("แม่น้ำอยู่ที่ไหน"),
+            ["แ", "ม่", "น้ํ", "า", "อ", "ยู่", "ที่", "ไ", "ห", "น"]
+        )
+        self.assertEqual(display_cell_tokenize("สวัสดี"), ['ส', 'วั', 'ส', 'ดี'])
+        self.assertEqual(display_cell_tokenize("ทดสอบ"), ["ท", "ด", "ส", "อ", "บ"])
+        self.assertEqual(display_cell_tokenize("ภาษาไทย"), ["ภ", "า", "ษ", "า", "ไ", "ท", "ย"])
diff --git a/tests/core/test_util.py b/tests/core/test_util.py
index f0a8e31b..5b8b12a9 100644
--- a/tests/core/test_util.py
+++ b/tests/core/test_util.py
@@ -32,6 +32,7 @@ from pythainlp.util import (
     ipa_to_rtgs,
     isthai,
     isthaichar,
+    longest_common_subsequence,
     nectec_to_ipa,
     normalize,
     now_reign_year,
@@ -842,3 +843,13 @@ class UtilTestCase(unittest.TestCase):
 
     # def test_abbreviation_to_full_text(self):
     #     self.assertIsInstance(abbreviation_to_full_text("รร.ของเราน่าอยู่", list))
+
+    def test_longest_common_subsequence(self):
+        self.assertEqual(longest_common_subsequence("ABCBDAB", "BDCAB"), "BDAB")
+        self.assertEqual(longest_common_subsequence("AGGTAB", "GXTXAYB"), "GTAB")
+        self.assertEqual(longest_common_subsequence("ABCDGH", "AEDFHR"), "ADH")
+        self.assertEqual(longest_common_subsequence("ABC", "AC"), "AC")
+        self.assertEqual(longest_common_subsequence("ABC", "DEF"), "")
+        self.assertEqual(longest_common_subsequence("", "ABC"), "")
+        self.assertEqual(longest_common_subsequence("ABC", ""), "")
+        self.assertEqual(longest_common_subsequence("", ""), "")