Repository: flairNLP/flair

Issue:
Add support for HIPE-2022 v2.1
Hi,

a new version 2.1 was recently released for the HIPE-2022 dataset.

Flair currently only supports version 2.0, so it needs to be updated.

Changelog for v2.1 release:

https://github.com/hipe-eval/HIPE-2022-data/releases/tag/v2.1


Discussion:

Resolving patch:
diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py
--- a/flair/datasets/sequence_labeling.py
+++ b/flair/datasets/sequence_labeling.py
@@ -4138,10 +4138,10 @@ class NER_HIPE_2022(ColumnCorpus):
     def _prepare_corpus(
         file_in: Path, file_out: Path, eos_marker: str, document_separator: str, add_document_separator: bool
     ):
-        with open(file_in, "rt") as f_p:
+        with open(file_in, "rt", encoding="utf-8") as f_p:
             lines = f_p.readlines()
 
-        with open(file_out, "wt") as f_out:
+        with open(file_out, "wt", encoding="utf-8") as f_out:
             # Add missing newline after header
             f_out.write(lines[0] + "\n")
 
@@ -4168,7 +4168,7 @@ def __init__(
         base_path: Union[str, Path] = None,
         tag_to_bioes: str = "ner",
         in_memory: bool = True,
-        version: str = "v2.0",
+        version: str = "v2.1",
         branch_name: str = "main",
         dev_split_name="dev",
         add_document_separator=False,
@@ -4220,6 +4220,8 @@ def __init__(
         hipe_available_splits["v2.0"] = hipe_available_splits["v1.0"].copy()
         hipe_available_splits["v2.0"]["ajmc"] = {"de": ["train", "dev"], "en": ["train", "dev"], "fr": ["train", "dev"]}
 
+        hipe_available_splits["v2.1"] = hipe_available_splits["v2.0"].copy()
+
         eos_marker = "EndOfSentence"
         document_separator = "# hipe2022:document_id"
 


Test patch:
diff --git a/tests/test_datasets.py b/tests/test_datasets.py
--- a/tests/test_datasets.py
+++ b/tests/test_datasets.py
@@ -1,3 +1,4 @@
+import copy
 import shutil
 
 import pytest
@@ -488,8 +489,14 @@ def test_hipe_2022_corpus(tasks_base_path):
             }
         }
     }
+    hipe_stats["v2.1"] = copy.deepcopy(hipe_stats["v2.0"])
+    hipe_stats["v2.1"]["hipe2020"]["fr"]["train"] = {
+        "sents": 5743,
+        "docs": 158,
+        "labels": ["loc", "org", "pers", "prod", "time"],
+    }
 
-    def test_hipe_2022(dataset_version="v1.0", add_document_separator=True):
+    def test_hipe_2022(dataset_version="v2.1", add_document_separator=True):
         for dataset_name, languages in hipe_stats[dataset_version].items():
             for language in languages:
                 splits = languages[language]
@@ -503,7 +510,7 @@ def test_hipe_2022(dataset_version="v1.0", add_document_separator=True):
                 )
 
                 for split_name, stats in splits.items():
-                    split_description = f"{dataset_name}/{language}@{split_name}"
+                    split_description = f"{dataset_name}@{dataset_version}/{language}#{split_name}"
 
                     current_sents = stats["sents"]
                     current_docs = stats["docs"]
@@ -557,6 +564,8 @@ def test_hipe_2022(dataset_version="v1.0", add_document_separator=True):
     test_hipe_2022(dataset_version="v1.0", add_document_separator=False)
     test_hipe_2022(dataset_version="v2.0", add_document_separator=True)
     test_hipe_2022(dataset_version="v2.0", add_document_separator=False)
+    test_hipe_2022(dataset_version="v2.1", add_document_separator=True)
+    test_hipe_2022(dataset_version="v2.1", add_document_separator=False)
 
 
 def test_multi_file_jsonl_corpus_should_use_label_type(tasks_base_path):