Repository: PyThaiNLP/pythainlp (python)

Issue:
Generalized reorder rules in text normalization
- Generalized `Nikhahit + Tonemark + Sara A` case to also cover `Nikhahit + Sara A`
  - New rule: `Nikhahit + TONEMARK* + Sara Aa -> TONEMARK* + Sara Am`
- Generalized `Sara Am + Tonemark` case to also cover follow vowels
  - New rule: `FOLLOWVOWEL+ + TONEMARK+ -> TONEMARK+ + FOLLOWVOWEL+`
- Generalized `Thanthakhat + Above/Follow vowel` and `Tonemark + Above/Follow vowel`
  -  New rule: `TONE/Thanthakhat+ + A/BVOWELV+ -> A/BVOWEL+ + TONE/Thanthakhat+`


Pull request description:
Improve normalize() function
This will close #372 

- Improve text normalization function
- Replace Thai characters with Unicode literals for readability



Patch:
diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py
index 7bccff38..f4fed0d7 100644
--- a/pythainlp/__init__.py
+++ b/pythainlp/__init__.py
@@ -2,26 +2,33 @@
 __version__ = "2.2.0-dev0"
 
 thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"  # 44 chars
+
 thai_vowels = (
-    "ฤฦะ\u0e31าำ\u0e34\u0e35\u0e36\u0e37\u0e38\u0e39เแโใไ\u0e45\u0e47"  # 19
-)
+    "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37"
+    + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47"
+)  # 20
+thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44"  # 5
+thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45"  # 4
+thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47"  # 7
+thai_below_vowels = "\u0e38\u0e39"  # 2
+
 thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b"  # 4
 
-# Thai Characters: Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan
+# Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan:
 # These signs can be part of a word
-thai_signs = "ฯๆ\u0e3a\u0e4c\u0e4d\u0e4e"  # 6 chars
+thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e"  # 6 chars
 
 # Any Thai character that can be part of a word
 thai_letters = "".join(
     [thai_consonants, thai_vowels, thai_tonemarks, thai_signs]
-)  # 73
+)  # 74
 
-# Thai Characters Fongman, Angkhankhu, Khomut
+# Fongman, Angkhankhu, Khomut:
 # These characters are section markers
 thai_punctuations = "\u0e4f\u0e5a\u0e5b"  # 3 chars
 
 thai_digits = "๐๑๒๓๔๕๖๗๘๙"  # 10
-thai_symbols = "฿"
+thai_symbols = "\u0e3f"  # Thai Bath ฿
 
 # All Thai characters that presented in Unicode
 thai_characters = "".join(
diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py
index f68d7d25..84138143 100644
--- a/pythainlp/util/normalize.py
+++ b/pythainlp/util/normalize.py
@@ -5,53 +5,44 @@ Text normalization
 import re
 import warnings
 
-from pythainlp import thai_tonemarks
-
-_NORMALIZE_RULE1 = [
-    "ะ",
-    "ั",
-    "็",
-    "า",
-    "ิ",
-    "ี",
-    "ึ",
-    "่",
-    "ํ",
-    "ุ",
-    "ู",
-    "ใ",
-    "ไ",
-    "โ",
-    "ื",
-    "่",
-    "้",
-    "๋",
-    "๊",
-    "ึ",
-    "์",
-    "๋",
-    "ำ",
-]  # เก็บพวกสระ วรรณยุกต์ที่ซ้ำกันแล้วมีปัญหา
-
-
-_NORMALIZE_RULE2 = [
-    ("เเ", "แ"),  # เ เ -> แ
-    ("ํา", "ำ"),  # นิคหิต + สระอา -> สระอำ
-    ("ํ(t)า", "\\1ำ"),
-    ("ํา(t)", "\\1ำ"),
-    ("([่-๋])([ัิ-ื])", "\\2\\1"),
-    ("([่-๋])([ูุ])", "\\2\\1"),
-    ("ำ([่-๋])", "\\1ำ"),
-    ("(์)([ัิ-ู])", "\\2\\1"),
-]  # เก็บพวก พิมพ์ลำดับผิดหรือผิดแป้นแต่กลับแสดงผลถูกต้อง ให้ไปเป็นแป้นที่ถูกต้อง เช่น เ + เ ไปเป็น แ
+from pythainlp import thai_above_vowels as above_v
+from pythainlp import thai_below_vowels as below_v
+from pythainlp import thai_follow_vowels as follow_v
+from pythainlp import thai_lead_vowels as lead_v
+from pythainlp import thai_tonemarks as tonemarks
+
+
+# VOWELS + Phinthu,Thanthakhat, Nikhahit, Yamakkan
+_NO_REPEAT_CHARS = (
+    f"{follow_v}{lead_v}{above_v}{below_v}\u0e3a\u0e4c\u0e4d\u0e4e"
+)
+_NORMALIZE_REPETITION = list(
+    zip([ch + "+" for ch in _NO_REPEAT_CHARS], _NO_REPEAT_CHARS)
+)
+
+_NORMALIZE_REORDER = [
+    ("\u0e40\u0e40", "\u0e41"),  # Sara E + Sara E -> Sara Ae
+    (
+        f"([{tonemarks}\u0e4c]+)([{above_v}{below_v}]+)",
+        "\\2\\1",
+    ),  # TONE/Thanthakhat+ + A/BVOWELV+ -> A/BVOWEL+ + TONE/Thanthakhat+
+    (
+        f"\u0e4d([{tonemarks}]*)\u0e32",
+        "\\1\u0e33",
+    ),  # Nikhahit + TONEMARK* + Sara Aa -> TONEMARK* + Sara Am
+    (
+        f"([{follow_v}]+)([{tonemarks}]+)",
+        "\\2\\1",
+    ),  # FOLLOWVOWEL+ + TONEMARK+ -> TONEMARK+ + FOLLOWVOWEL+
+]
 
 
 def normalize(text: str) -> str:
     """
     This function normalize thai text with normalizing rules as follows:
 
-        * Remove redudant symbol of tones and vowels.
-        * Subsitute ["เ", "เ"] to "แ".
+        * Remove redundant vowels and tonemarks
+        * Subsitute "เ" + "เ" with "แ"
 
     :param str text: thai text to be normalized
     :return: normalized Thai text according to the fules
@@ -71,10 +62,10 @@ def normalize(text: str) -> str:
         normalize('นานาาา')
         # output: นานา
     """
-    for data in _NORMALIZE_RULE2:
-        text = re.sub(data[0].replace("t", "[่้๊๋]"), data[1], text)
-    for data in list(zip(_NORMALIZE_RULE1, _NORMALIZE_RULE1)):
-        text = re.sub(data[0].replace("t", "[่้๊๋]") + "+", data[1], text)
+    for data in _NORMALIZE_REORDER:
+        text = re.sub(data[0], data[1], text)
+    for data in _NORMALIZE_REPETITION:
+        text = re.sub(data[0], data[1], text)
     return text
 
 
@@ -100,7 +91,7 @@ def delete_tone(text: str) -> str:
         delete_tone('สองพันหนึ่งร้อยสี่สิบเจ็ดล้านสี่แสนแปดหมื่นสามพันหกร้อยสี่สิบเจ็ด')
         # output: สองพันหนึงรอยสีสิบเจ็ดลานสีแสนแปดหมืนสามพันหกรอยสีสิบเจ็ด
     """
-    chars = [ch for ch in text if ch not in thai_tonemarks]
+    chars = [ch for ch in text if ch not in tonemarks]
     return "".join(chars)
 
 


Test patch:
diff --git a/tests/test_util.py b/tests/test_util.py
index 3ba011e3..585a29e9 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -255,9 +255,30 @@ class TestUtilPackage(unittest.TestCase):
         self.assertEqual(deletetone("จิ้น"), delete_tone("จิ้น"))
 
     def test_normalize(self):
+        self.assertIsNotNone(normalize("พรรค์จันทร์ab์"))
+
+        # sara e + sara e
         self.assertEqual(normalize("เเปลก"), "แปลก")
+
+        # consonant + follow vowel + tonemark
+        self.assertEqual(normalize("\u0e01\u0e30\u0e48"), "\u0e01\u0e48\u0e30")
+
+        # consonant + nikhahit + sara aa
         self.assertEqual(normalize("นํา"), "นำ")
-        self.assertIsNotNone(normalize("พรรค์จันทร์ab์"))
+        self.assertEqual(normalize("\u0e01\u0e4d\u0e32"), "\u0e01\u0e33")
+
+        # consonant + nikhahit + tonemark + sara aa
+        self.assertEqual(
+            normalize("\u0e01\u0e4d\u0e48\u0e32"), "\u0e01\u0e48\u0e33"
+        )
+
+        # consonant + tonemark + nikhahit + sara aa
+        self.assertEqual(
+            normalize("\u0e01\u0e48\u0e4d\u0e32"), "\u0e01\u0e48\u0e33"
+        )
+
+        # consonant + follow vowel + tonemark
+        self.assertEqual(normalize("\u0e01\u0e32\u0e48"), "\u0e01\u0e48\u0e32")
 
     # ### pythainlp.util.thai
 
@@ -329,13 +350,16 @@ class TestUtilPackage(unittest.TestCase):
             now + datetime.timedelta(days=0), thai_day2datetime("วันนี้", now)
         )
         self.assertEqual(
-            now + datetime.timedelta(days=1), thai_day2datetime("พรุ่งนี้", now)
+            now + datetime.timedelta(days=1),
+            thai_day2datetime("พรุ่งนี้", now),
         )
         self.assertEqual(
-            now + datetime.timedelta(days=2), thai_day2datetime("มะรืนนี้", now)
+            now + datetime.timedelta(days=2),
+            thai_day2datetime("มะรืนนี้", now),
         )
         self.assertEqual(
-            now + datetime.timedelta(days=-1), thai_day2datetime("เมื่อวาน", now)
+            now + datetime.timedelta(days=-1),
+            thai_day2datetime("เมื่อวาน", now),
         )
         self.assertEqual(
             now + datetime.timedelta(days=-2), thai_day2datetime("วานซืน", now)