Repository: flairNLP/flair

Issue:
Add support for new MasakhaNER v2 dataset
Hi,

MasakhaNER v2 was recently [accepted at EMNLP 20220](https://twitter.com/davlanade/status/1578010346157400064) and the new dataset is already online [available here](https://github.com/masakhane-io/masakhane-ner/tree/main/MasakhaNER2.0/data).

Preprint is available [here](https://arxiv.org/abs/2210.12391).

It should be relatively easy to add this dataset.

The current existing v1 has the following arguments:

https://github.com/flairNLP/flair/blob/8d27a383810455cb45d650e7fe2003384780ae84/flair/datasets/sequence_labeling.py#L2550-L2557

I think we can simply add a `version` variable and default-set it to `v1` to ensure backward compatibility?

Then version dependend-logic such as available languages and GitHub folder paths could be added.



Discussion:
This approach would avoid adding a new dataset called e.g. `NER_MASAKHANE_v2` which includes duplicated code. What do you thin @alanakbik  :thinking: 
@dadelani told me that one language (Luo, v2) is currently missing due to license issues. So 19/20 languages can be used.

Table 1 of the preprint also includes the number of sentences per split for each language, so it should be relatively easy to also include unit tests. 
@stefan-it that's great! Yes, I would reuse the old object and add a version parameter. Since V2 will probably be the standard moving forward, perhaps by default it should be set to `v2`?

Resolving patch:
diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py
--- a/flair/datasets/sequence_labeling.py
+++ b/flair/datasets/sequence_labeling.py
@@ -2559,6 +2559,7 @@ class NER_MASAKHANE(MultiCorpus):
     def __init__(
         self,
         languages: Union[str, List[str]] = "luo",
+        version: str = "v2",
         base_path: Union[str, Path] = None,
         in_memory: bool = True,
         **corpusargs,
@@ -2567,6 +2568,7 @@ def __init__(
         Initialize the Masakhane corpus available on https://github.com/masakhane-io/masakhane-ner/tree/main/data.
         It consists of ten African languages. Pass a language code or a list of language codes to initialize the corpus
         with the languages you require. If you pass "all", all languages will be initialized.
+        :version: Specifies version of the dataset. Currently, only "v1" and "v2" are supported, using "v2" as default.
         :param base_path: Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
         to point to a different folder but typically this should not be necessary.
         POS tags instead
@@ -2587,19 +2589,56 @@ def __init__(
         # this dataset name
         dataset_name = self.__class__.__name__.lower()
 
-        data_folder = base_path / dataset_name
+        supported_versions = ["v1", "v2"]
+
+        if version not in supported_versions:
+            log.error(f"The specified version '{version}' is not in the list of supported version!")
+            log.error(f"Supported versions are '{supported_versions}'!")
+            raise Exception
+
+        data_folder = base_path / dataset_name / version
+
+        languages_to_code = {
+            "v1": {
+                "amharic": "amh",
+                "hausa": "hau",
+                "igbo": "ibo",
+                "kinyarwanda": "kin",
+                "luganda": "lug",
+                "luo": "luo",
+                "naija": "pcm",
+                "swahili": "swa",
+                "yoruba": "yor",
+                "wolof": "wol",
+            },
+            "v2": {
+                "bambara": "bam",
+                "ghomala": "bbj",
+                "ewe": "ewe",
+                "fon": "fon",
+                "hausa": "hau",
+                "igbo": "ibo",
+                "kinyarwanda": "kin",
+                "luganda": "lug",
+                "mossi": "mos",
+                "naija": "pcm",
+                "chichewa": "nya",
+                "chishona": "sna",
+                "kiswahili": "swa",
+                "setswana": "tsn",
+                "akan_twi": "twi",
+                "wolof": "wol",
+                "isixhosa": "xho",
+                "yoruba": "yor",
+                "isizulu": "zul",
+            },
+        }
 
-        language_to_code = {
-            "amharic": "amh",
-            "hausa": "hau",
-            "igbo": "ibo",
-            "kinyarwanda": "kin",
-            "luganda": "lug",
-            "luo": "luo",
-            "naija": "pcm",
-            "swahili": "swa",
-            "yoruba": "yor",
-            "wolof": "wol",
+        language_to_code = languages_to_code[version]
+
+        data_paths = {
+            "v1": "https://raw.githubusercontent.com/masakhane-io/masakhane-ner/main/data",
+            "v2": "https://raw.githubusercontent.com/masakhane-io/masakhane-ner/main/MasakhaNER2.0/data",
         }
 
         # use all languages if explicitly set to "all"
@@ -2621,13 +2660,13 @@ def __init__(
             language_folder = data_folder / language
 
             # download data if necessary
-            data_path = f"https://raw.githubusercontent.com/masakhane-io/masakhane-ner/main/data/{language}/"
+            data_path = f"{data_paths[version]}/{language}/"
             cached_path(f"{data_path}dev.txt", language_folder)
             cached_path(f"{data_path}test.txt", language_folder)
             cached_path(f"{data_path}train.txt", language_folder)
 
             # initialize comlumncorpus and add it to list
-            log.info(f"Reading data for language {language}")
+            log.info(f"Reading data for language {language}@{version}")
             corp = ColumnCorpus(
                 data_folder=language_folder,
                 column_format=columns,


Test patch:
diff --git a/tests/test_datasets.py b/tests/test_datasets.py
--- a/tests/test_datasets.py
+++ b/tests/test_datasets.py
@@ -703,6 +703,91 @@ def check_number_sentences(reference: int, actual: int, split_name: str):
         check_number_sentences(len(corpus.test), gold_stats[language]["test"], "test")
 
 
+def test_masakhane_corpus(tasks_base_path):
+    """
+    This test covers the complete MasakhaNER dataset, including support for v1 and v2.
+    """
+    supported_versions = ["v1", "v2"]
+
+    supported_languages = {
+        "v1": ["amh", "hau", "ibo", "kin", "lug", "luo", "pcm", "swa", "wol", "yor"],
+        "v2": [
+            "bam",
+            "bbj",
+            "ewe",
+            "fon",
+            "hau",
+            "ibo",
+            "kin",
+            "lug",
+            "mos",
+            "pcm",
+            "nya",
+            "sna",
+            "swa",
+            "tsn",
+            "twi",
+            "wol",
+            "xho",
+            "yor",
+            "zul",
+        ],
+    }
+
+    masakhane_stats = {
+        "v1": {
+            "amh": {"train": 1750, "dev": 250, "test": 500},
+            "hau": {"train": 1912, "dev": 276, "test": 552},
+            "ibo": {"train": 2235, "dev": 320, "test": 638},
+            "kin": {
+                "train": 2116,
+                "dev": 302,
+                "test": 605,
+            },
+            "lug": {"train": 1428, "dev": 200, "test": 407},
+            "luo": {"train": 644, "dev": 92, "test": 186},
+            "pcm": {"train": 2124, "dev": 306, "test": 600},
+            "swa": {"train": 2109, "dev": 300, "test": 604},
+            "wol": {"train": 1871, "dev": 267, "test": 539},
+            "yor": {"train": 2171, "dev": 305, "test": 645},
+        },
+        "v2": {
+            "bam": {"train": 4462, "dev": 638, "test": 1274},
+            "bbj": {"train": 3384, "dev": 483, "test": 966},
+            "ewe": {"train": 3505, "dev": 501, "test": 1001},
+            "fon": {"train": 4343, "dev": 621, "test": 1240},
+            "hau": {"train": 5716, "dev": 816, "test": 1633},
+            "ibo": {"train": 7634, "dev": 1090, "test": 2181},
+            "kin": {"train": 7825, "dev": 1118, "test": 2235},
+            "lug": {"train": 4942, "dev": 706, "test": 1412},
+            "mos": {"train": 4532, "dev": 648, "test": 1294},
+            "pcm": {"train": 5646, "dev": 806, "test": 1613},
+            "nya": {"train": 6250, "dev": 893, "test": 1785},
+            "sna": {"train": 6207, "dev": 887, "test": 1773},
+            "swa": {"train": 6593, "dev": 942, "test": 1883},
+            "tsn": {"train": 3489, "dev": 499, "test": 996},
+            "twi": {"train": 4240, "dev": 605, "test": 1211},
+            "wol": {"train": 4593, "dev": 656, "test": 1312},
+            "xho": {"train": 5718, "dev": 817, "test": 1633},
+            "yor": {"train": 6876, "dev": 983, "test": 1964},
+            "zul": {"train": 5848, "dev": 836, "test": 1670},
+        },
+    }
+
+    def check_number_sentences(reference: int, actual: int, split_name: str, language: str, version: str):
+        assert actual == reference, f"Mismatch in number of sentences for {language}@{version}/{split_name}"
+
+    for version in supported_versions:
+        for language in supported_languages[version]:
+            corpus = flair.datasets.NER_MASAKHANE(languages=language, version=version)
+
+            gold_stats = masakhane_stats[version][language]
+
+            check_number_sentences(len(corpus.train), gold_stats["train"], "train", language, version)
+            check_number_sentences(len(corpus.dev), gold_stats["dev"], "dev", language, version)
+            check_number_sentences(len(corpus.test), gold_stats["test"], "test", language, version)
+
+
 def test_multi_file_jsonl_corpus_should_use_label_type(tasks_base_path):
     corpus = MultiFileJsonlCorpus(
         train_files=[tasks_base_path / "jsonl/train.jsonl"],