Repository: tatuylonen/wiktextract

Issue:
[en] ancient greek articles treated as separate forms
Ancient greek forms with comma-separated articles (e.g. [ἰχθυοκένταυρος](https://en.wiktionary.org/wiki/%E1%BC%B0%CF%87%CE%B8%CF%85%CE%BF%CE%BA%CE%AD%CE%BD%CF%84%CE%B1%CF%85%CF%81%CE%BF%CF%82#Ancient%20Greek)), result in the first article being treated as a separate form,  [ἰχθυοκένταυρος on kaikki](https://kaikki.org/dictionary/All%20languages%20combined/meaning/%E1%BC%B0/%E1%BC%B0%CF%87/%E1%BC%B0%CF%87%CE%B8%CF%85%CE%BF%CE%BA%CE%AD%CE%BD%CF%84%CE%B1%CF%85%CF%81%CE%BF%CF%82.html):

```json
{
      "form": "ὁ",
      "roman": "ho",
      "source": "declension",
      "tags": [
        "nominative",
        "singular"
      ]
}
```

Hints from issue comments:
@kristian-clausal: Augh, this turned out to be a rabbit-hole...

I found a piece of config-data in lang_specific that seemed like it would be perfect for adding regexes that remove the articles from the start of the forms.

... But it was unimplemented??

After faffing about for too damn long, turns out it was me, I started on this for some kludge or other three years ago, implemented a thing I needed it for and then... forgot to add the rest.

So now I added some of the rest, just to use it for this.

The fix in #1107 removes the articles (which I will assume are predictable and regular, seemed like it was just gendered articles).

Solution patch:
diff --git a/src/wiktextract/extractor/en/inflection.py b/src/wiktextract/extractor/en/inflection.py
index 472495d9..14a04eac 100644
--- a/src/wiktextract/extractor/en/inflection.py
+++ b/src/wiktextract/extractor/en/inflection.py
@@ -1691,6 +1691,7 @@ def parse_simple_table(
                     for k, v in repls.items():
                         alt = re.sub(k, v, alt)
                     alts.append(alt)
+
         # Remove "*" from beginning of forms, as in non-attested
         # or reconstructed forms.  Otherwise it might confuse romanization
         # detection.
@@ -2187,6 +2188,7 @@ def parse_simple_table(
     table_tags = []
     special_phrase_splits = get_lang_conf(lang, "special_phrase_splits")
     form_replacements = get_lang_conf(lang, "form_replacements")
+    form_transformations = get_lang_conf(lang, "form_transformations")
     possibly_ignored_forms = get_lang_conf(lang, "conditionally_ignored_cells")
     cleanup_rules = get_lang_conf(lang, "minor_text_cleanups")
 
@@ -2505,6 +2507,30 @@ def parse_simple_table(
                     assert isinstance(tags, str)
                     form = replacement
                     extra_tags.extend(tags.split())
+
+                check_romanization_form_transformation = False
+                # loop over regexes in form_transformation and replace text
+                # in form using regex patterns
+                # this does a bit of the same stuff the above does,
+                # but with regexes and re.sub() instead
+                for (
+                    form_transformations_pos,
+                    v,
+                    subst,
+                    tags,
+                ) in form_transformations:
+                    # v is a pattern string, like "^ich"
+                    if pos != form_transformations_pos:
+                        continue
+                    m = re.search(v, form)
+                    if m is not None:
+                        form = re.sub(v, subst, form)
+                        for x in tags.split():
+                            assert x in valid_tags
+                        extra_tags.extend(tags.split())
+                        check_romanization_form_transformation = True
+                        break
+
                 # Clean the value, extracting reference symbols
                 form, refs, defs, hdr_tags = extract_cell_content(
                     lang, word, form
@@ -2520,6 +2546,23 @@ def parse_simple_table(
                         refs_tags.update(def_ht[ref])
 
                 if base_roman:
+                    if check_romanization_form_transformation:
+                        # because form_transformations are used to handle things
+                        # where the romanization has the "same" structure, we
+                        # need to handle that here too....
+                        for (
+                            _,
+                            v,
+                            subst,
+                            _,
+                        ) in form_transformations:
+                            # v is a pattern string, like "^ich"
+                            m = re.search(v, base_roman)
+                            if m is not None:
+                                base_roman = re.sub(v, subst, base_roman)
+                                # XXX add tag stuff here if needed
+                                break
+
                     base_roman, _, _, hdr_tags = extract_cell_content(
                         lang, word, base_roman
                     )
diff --git a/src/wiktextract/extractor/en/lang_specific_configs.py b/src/wiktextract/extractor/en/lang_specific_configs.py
index a8aae4cb..be0b567a 100644
--- a/src/wiktextract/extractor/en/lang_specific_configs.py
+++ b/src/wiktextract/extractor/en/lang_specific_configs.py
@@ -183,6 +183,32 @@ lang_specific: dict[str, LangConfDict] = {
     },
     "Ancient Greek": {
         "next": "Proto-Indo-European",  # Has dual
+        "form_transformations": [
+            # Used to remove the gendered article alternatives at the start
+            # of table entries like ἰχθυοκένταυρος / Ancient Greek
+            ["noun", "^ὁ, ἡ ", "", ""],
+            ["noun", "^τὼ ", "", ""],
+            ["noun", "^οἱ, αἱ ", "", ""],
+            ["noun", "^τοῦ, τῆς ", "", ""],
+            ["noun", "^τοῖν ", "", ""],
+            ["noun", "^τῶν ", "", ""],
+            ["noun", "^τῷ, τῇ ", "", ""],
+            ["noun", "^τοῖς, ταῖς ", "", ""],
+            ["noun", "^τὸν, τὴν ", "", ""],
+            ["noun", "^τὼ ", "", ""],
+            ["noun", "^τοὺς, τᾱ̀ς ", "", ""],
+            ["noun", "(?m)^ho, hē ", "", ""],
+            ["noun", "(?m)^tṑ ", "", ""],
+            ["noun", "(?m)^hoi, hai ", "", ""],
+            ["noun", "(?m)^toû, tês", "", ""],
+            ["noun", "(?m)^toîn ", "", ""],
+            ["noun", "(?m)^tôn ", "", ""],
+            ["noun", "(?m)^tôi, têi ", "", ""],
+            ["noun", "(?m)^toîs, taîs ", "", ""],
+            ["noun", "(?m)^tòn, tḕn ", "", ""],
+            ["noun", "(?m)^tṑ ", "", ""],
+            ["noun", "(?m)^toùs, tā̀s ", "", ""],
+        ],
     },
     # "Anejom̃": {
     #     "numbers": ["singular", "dual", "trial", "plural"],
@@ -285,8 +311,7 @@ lang_specific: dict[str, LangConfDict] = {
             ["verb", r"^you ", "", "second-person"],
             ["verb", r"^he ", "", "third-person singular"],
             ["verb", r"^we ", "", "first-person plural"],
-            ["verb", r"^you ", "", "second-person plural"],
-            ["verb", r"^they ", "", "third-person plural"],
+            ["verb", r"^they ", "", "third-person"],
             ["verb", r"^it ", "", "third-person singular"],
             ["verb", r"^thou ", "", "second-person singular"],
             ["verb", r"^ye ", "", "second-person plural"],
@@ -295,7 +320,7 @@ lang_specific: dict[str, LangConfDict] = {
             ["verb", r"^he/she/it ", "", "third-person singular"],
             ["verb", r"^he/she/it/they ", "", "third-person singular"],
             ["verb", r"\bhim/her/it/them ", "", "third-person singular"],
-            ["verb", r"\bthem ", "", "third-person plural"],
+            ["verb", r"\bthem ", "", "third-person"],
             ["verb", r"\bus ", "", "first-person plural"],
             ["verb", r"\bme ", "", "first-person singular"],
         ],


Test patch:
diff --git a/tests/test_en_inflection_en.py b/tests/test_en_inflection_en.py
index b490acf6..4fb3d689 100644
--- a/tests/test_en_inflection_en.py
+++ b/tests/test_en_inflection_en.py
@@ -865,7 +865,7 @@ class InflTests(unittest.TestCase):
                 {
                     "form": "affect",
                     "source": "Conjugation",
-                    "tags": ["plural", "present", "third-person"],
+                    "tags": ["present", "third-person"],
                 },
                 {
                     "form": "is affecting",
@@ -882,7 +882,6 @@ class InflTests(unittest.TestCase):
                     "form": "are affecting",
                     "source": "Conjugation",
                     "tags": [
-                        "plural",
                         "present",
                         "progressive",
                         "third-person",
@@ -905,7 +904,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "perfect",
-                        "plural",
                         "present",
                         "third-person",
                         "multiword-construction",
@@ -928,7 +926,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "perfect",
-                        "plural",
                         "present",
                         "progressive",
                         "third-person",
@@ -1057,7 +1054,7 @@ class InflTests(unittest.TestCase):
                 {
                     "form": "affected",
                     "source": "Conjugation",
-                    "tags": ["past", "plural", "third-person"],
+                    "tags": ["past", "third-person"],
                 },
                 {
                     "form": "was affecting",
@@ -1075,7 +1072,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "past",
-                        "plural",
                         "progressive",
                         "third-person",
                         "multiword-construction",
@@ -1098,7 +1094,6 @@ class InflTests(unittest.TestCase):
                     "tags": [
                         "past",
                         "perfect",
-                        "plural",
                         "third-person",
                         "multiword-construction",
                     ],
@@ -1121,7 +1116,6 @@ class InflTests(unittest.TestCase):
                     "tags": [
                         "past",
                         "perfect",
-                        "plural",
                         "progressive",
                         "third-person",
                         "multiword-construction",
@@ -1270,7 +1264,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "future",
-                        "plural",
                         "third-person",
                         "multiword-construction",
                     ],
@@ -1291,7 +1284,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "future",
-                        "plural",
                         "progressive",
                         "third-person",
                         "multiword-construction",
@@ -1314,7 +1306,6 @@ class InflTests(unittest.TestCase):
                     "tags": [
                         "future",
                         "perfect",
-                        "plural",
                         "third-person",
                         "multiword-construction",
                     ],
@@ -1337,7 +1328,6 @@ class InflTests(unittest.TestCase):
                     "tags": [
                         "future",
                         "perfect",
-                        "plural",
                         "progressive",
                         "third-person",
                         "multiword-construction",
@@ -1486,7 +1476,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "conditional",
-                        "plural",
                         "third-person",
                         "multiword-construction",
                     ],
@@ -1507,7 +1496,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "conditional",
-                        "plural",
                         "progressive",
                         "third-person",
                         "multiword-construction",
@@ -1530,7 +1518,6 @@ class InflTests(unittest.TestCase):
                     "tags": [
                         "conditional",
                         "perfect",
-                        "plural",
                         "third-person",
                         "multiword-construction",
                     ],
@@ -1553,7 +1540,6 @@ class InflTests(unittest.TestCase):
                     "tags": [
                         "conditional",
                         "perfect",
-                        "plural",
                         "progressive",
                         "third-person",
                         "multiword-construction",
@@ -2395,7 +2381,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "indicative",
-                        "plural",
                         "present",
                         "singular",
                         "third-person",
@@ -2406,7 +2391,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "indicative",
-                        "plural",
                         "present",
                         "singular",
                         "third-person",
@@ -2566,7 +2550,6 @@ class InflTests(unittest.TestCase):
                     "source": "Conjugation",
                     "tags": [
                         "indicative",
-                        "plural",
                         "preterite",
                         "singular",
                         "third-person",