Repository: flairNLP/flair

Issue:
BIOMEDICAL DATASETS: UnicodeDecodeError: Error loading several biomedical datasets
**Describe the bug**
---------------------------------------------------------------------------
UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 5851: character maps to <undefined>

This is a Unicode error that occurs when loading biomedical datasets. I have the following error when trying to load HUNER_CHEMICAL_CEMP(). 

**To Reproduce**
from flair.datasets import HUNER_CHEMICAL_CEMP
corpus = HUNER_CHEMICAL_CEMP()

**Expected behavior**
Corpus should load properly.

**Screenshots**
--------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-1-33d42c5cb73e> in <module>
      2 # # # 1. get all corpora for a specific entity type
      3 # # from flair.models import SequenceTagger
----> 4 corpus = HUNER_CHEMICAL_CEMP()

~\AppData\Local\Continuum\anaconda3\envs\fgpu\lib\site-packages\flair\datasets\biomedical.py in __init__(self, *args, **kwargs)
   3999 
   4000     def __init__(self, *args, **kwargs):
-> 4001         super().__init__(*args, **kwargs)
   4002 
   4003     @staticmethod

~\AppData\Local\Continuum\anaconda3\envs\fgpu\lib\site-packages\flair\datasets\biomedical.py in __init__(self, base_path, in_memory, sentence_splitter)
    522 
    523             writer = CoNLLWriter(sentence_splitter=self.sentence_splitter)
--> 524             internal_dataset = self.to_internal(data_folder)
    525 
    526             train_data = self.get_subset(internal_dataset, "train", splits_dir)

~\AppData\Local\Continuum\anaconda3\envs\fgpu\lib\site-packages\flair\datasets\biomedical.py in to_internal(self, data_dir)
   4009         train_text_file = train_folder / "chemdner_patents_train_text.txt"
   4010         train_ann_file = train_folder / "chemdner_cemp_gold_standard_train.tsv"
-> 4011         train_data = CEMP.parse_input_file(train_text_file, train_ann_file)
   4012 
   4013         dev_folder = CEMP.download_dev_corpus(data_dir)

~\AppData\Local\Continuum\anaconda3\envs\fgpu\lib\site-packages\flair\datasets\biomedical.py in parse_input_file(text_file, ann_file)
   3956 
   3957         with open(str(text_file), "r") as text_reader:
-> 3958             for line in text_reader:
   3959                 if not line:
   3960                     continue

~\AppData\Local\Continuum\anaconda3\envs\fgpu\lib\encodings\cp1252.py in decode(self, input, final)
     21 class IncrementalDecoder(codecs.IncrementalDecoder):
     22     def decode(self, input, final=False):
---> 23         return codecs.charmap_decode(input,self.errors,decoding_table)[0]
     24 
     25 class StreamWriter(Codec,codecs.StreamWriter):

UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 5851: character maps to <undefined>

**Environment (please complete the following information):**
---------------------------------------------------------------------------
 - OS [Windows]:
 - Version [e.g. flair-0.6]:

**Additional context**
---------------------------------------------------------------------------
Infact, except for HUNER_CHEMICAL_CDR(), HUNER_DISEASE_CDR(), HUNER_DISEASE_NCBI(), HUNER_DISEASE_SCAI(), 
I am getting similar Unicode Error when loading all other chemical (HUNER_CHEMICAL_CEMP(), HUNER_CHEMICAL_CHEBI(), HUNER_CHEMICAL_CHEMDNER(), HUNER_CHEMICAL_SCAI()) and disease (HUNER_DISEASE_MIRNA(), HUNER_DISEASE_VARIOME()) datasets.

** Similar error occurs when loading datasets as shown below.
from flair.datasets import CEMP
corpus = CEMP()
### dataset name can be replaced with any other datasets mentioned above and similar Unicode error occurs.


Discussion:
Thanks for reporting this @datasri . I will look into this in the next couple of days.
Thanks @leonweber

Resolving patch:
diff --git a/flair/datasets/biomedical.py b/flair/datasets/biomedical.py
--- a/flair/datasets/biomedical.py
+++ b/flair/datasets/biomedical.py
@@ -312,11 +312,11 @@ def brat_to_internal(corpus_dir: Path, ann_file_suffixes=None) -> InternalBioNer
     documents = {}
     entities_per_document = defaultdict(list)
     for text_file in text_files:
-        document_text = open(str(text_file)).read().strip()
+        document_text = open(str(text_file), encoding="utf8").read().strip()
         document_id = text_file.stem
 
         for suffix in ann_file_suffixes:
-            with open(str(text_file.with_suffix(suffix)), "r") as ann_file:
+            with open(str(text_file.with_suffix(suffix)), "r", encoding="utf8") as ann_file:
                 for line in ann_file:
                     fields = line.strip().split("\t")
 
@@ -374,7 +374,7 @@ def write_to_conll(self, dataset: InternalBioNerDataset, output_file: Path):
         os.makedirs(str(output_file.parent), exist_ok=True)
         filter_nested_entities(dataset)
 
-        with output_file.open("w") as f:
+        with output_file.open("w", encoding="utf8") as f:
             for document_id in Tqdm.tqdm(
                 dataset.documents.keys(),
                 total=len(dataset.documents),
@@ -545,7 +545,7 @@ def __init__(
     def get_subset(self, dataset: InternalBioNerDataset, split: str, split_dir: Path):
         split_file = cached_path(f"{self.split_url()}.{split}", split_dir)
 
-        with split_file.open() as f:
+        with split_file.open(encoding="utf8") as f:
             ids = [l.strip() for l in f if l.strip()]
             ids = sorted(id_ for id_ in ids if id_ in dataset.documents)
 
@@ -800,7 +800,7 @@ def read_file(
         documents = {}
         entities_per_document = defaultdict(list)
 
-        with open(str(input_iob_file), "r") as file_reader:
+        with open(str(input_iob_file), "r", encoding="utf8") as file_reader:
             document_id = None
             document_text = None
 
@@ -995,7 +995,7 @@ def read_folder(cls, data_folder: Path) -> InternalBioNerDataset:
         documents = {}
         entities_per_document = defaultdict(list)
         for ann_file in ann_files:
-            with ann_file.open() as f_ann, ann_file.with_suffix(".txt").open() as f_txt:
+            with ann_file.open(encoding="utf8") as f_ann, ann_file.with_suffix(".txt").open(encoding="utf8") as f_txt:
                 document_text = f_txt.read().strip()
 
                 document_id = ann_file.stem
@@ -1205,7 +1205,7 @@ def get_mirna_subset(
     ):
         split_file = cached_path(split_url, split_dir)
 
-        with split_file.open() as f:
+        with split_file.open(encoding="utf8") as f:
             ids = [l.strip() for l in f if l.strip()]
             ids = [id + "-train" for id in ids] + [id + "-test" for id in ids]
             ids = sorted(id_ for id_ in ids if id_ in dataset.documents)
@@ -1355,7 +1355,7 @@ def prepare_and_save_dataset(nersuite_folder: Path, output_file: Path):
                 continue
 
             annotations = []
-            with open(os.path.join(str(nersuite_folder), file), "r") as reader:
+            with open(os.path.join(str(nersuite_folder), file), "r", encoding="utf8") as reader:
                 for line in reader.readlines():
                     columns = line.split("\t")
                     annotations.append(columns[:4])
@@ -1408,7 +1408,7 @@ def read_dataset(
 
             document_id = file.replace(".nersuite", "")
 
-            with open(os.path.join(str(nersuite_folder), file), "r") as reader:
+            with open(os.path.join(str(nersuite_folder), file), "r", encoding="utf8") as reader:
                 document_text = ""
                 entities = []
 
@@ -1687,7 +1687,7 @@ def parse_dataset(data_dir: Path) -> InternalBioNerDataset:
             document_id = file.strip(".json")
             entities = []
 
-            with open(os.path.join(str(loctext_json_folder), file), "r") as f_in:
+            with open(os.path.join(str(loctext_json_folder), file), "r", encoding="utf8") as f_in:
                 data = json.load(f_in)
                 document_text = data["text"].strip()
                 document_text = document_text.replace("\n", " ")
@@ -2032,12 +2032,12 @@ def download_and_parse_dataset(data_dir: Path):
         for filename in os.listdir(str(texts_directory)):
             document_id = filename.strip(".txt")
 
-            with open(os.path.join(str(texts_directory), filename), "r") as file:
+            with open(os.path.join(str(texts_directory), filename), "r", encoding="utf8") as file:
                 documents[document_id] = file.read().strip()
 
         # Read annotations
         tag_file = data_dir / "manual-corpus-species-1.0" / "filtered_tags.tsv"
-        with open(str(tag_file), "r") as file:
+        with open(str(tag_file), "r", encoding="utf8") as file:
             next(file)  # Ignore header row
 
             for line in file:
@@ -2475,8 +2475,8 @@ def patch_training_file(orig_train_file: Path, patched_file: Path):
         patch_lines = {
             3249: '10923035\t711\t761\tgeneralized epilepsy and febrile seizures " plus "\tSpecificDisease\tD004829+D003294\n'
         }
-        with open(str(orig_train_file), "r") as input:
-            with open(str(patched_file), "w") as output:
+        with open(str(orig_train_file), "r", encoding="utf8") as input:
+            with open(str(patched_file), "w", encoding="utf8") as output:
                 line_no = 1
 
                 for line in input:
@@ -2490,7 +2490,7 @@ def parse_input_file(input_file: Path):
         documents = {}
         entities_per_document = {}
 
-        with open(str(input_file), "r") as file:
+        with open(str(input_file), "r", encoding="utf8") as file:
             document_id = None
             document_text = None
             entities = []
@@ -2854,7 +2854,7 @@ def parse_dataset(cls, corpus_folder: Path, fix_annotation=True):
         ]
         for text_file in input_files:
 
-            with open(os.path.join(str(corpus_folder), text_file)) as text_reader:
+            with open(os.path.join(str(corpus_folder), text_file), encoding="utf8") as text_reader:
                 document_text = text_reader.read()
                 if not document_text:
                     continue
@@ -2864,7 +2864,7 @@ def parse_dataset(cls, corpus_folder: Path, fix_annotation=True):
                 text_offset = document_text.find(article_parts[1])
                 document_text = (article_parts[1] + "  " + article_parts[2]).strip()
 
-            with open(os.path.join(str(corpus_folder), text_file + ".ann")) as ann_file:
+            with open(os.path.join(str(corpus_folder), text_file + ".ann"), encoding="utf8") as ann_file:
                 entities = []
 
                 tree = etree.parse(ann_file)
@@ -2981,7 +2981,7 @@ def download_dataset(data_dir: Path):
     def parse_dataset(data_dir: Path) -> InternalBioNerDataset:
         entities_per_document = defaultdict(list)
         texts_per_document = {}
-        with (data_dir / "S800.tsv").open() as f:
+        with (data_dir / "S800.tsv").open(encoding="utf8") as f:
             for line in f:
                 fields = line.strip().split("\t")
                 if not fields:
@@ -2995,7 +2995,7 @@ def parse_dataset(data_dir: Path) -> InternalBioNerDataset:
                 entities_per_document[fname].append(Entity((start, end), "Species"))
 
         for fname in entities_per_document:
-            with (data_dir / "abstracts" / fname).with_suffix(".txt").open() as f:
+            with (data_dir / "abstracts" / fname).with_suffix(".txt").open(encoding="utf8") as f:
                 texts_per_document[fname] = f.read()
 
         return InternalBioNerDataset(
@@ -3111,7 +3111,7 @@ def parse_input_file(text_file: Path, ann_file: Path) -> InternalBioNerDataset:
 
         document_title_length = {}
 
-        with open(str(text_file), "r") as text_reader:
+        with open(str(text_file), "r", encoding="utf8") as text_reader:
             for line in text_reader:
                 if not line:
                     continue
@@ -3122,7 +3122,7 @@ def parse_input_file(text_file: Path, ann_file: Path) -> InternalBioNerDataset:
 
                 entities_per_document[document_id] = []
 
-        with open(str(ann_file), "r") as ann_reader:
+        with open(str(ann_file), "r", encoding="utf8") as ann_reader:
             for line in ann_reader:
                 if not line:
                     continue
@@ -3246,11 +3246,11 @@ def parse_corpus(text_dir: Path, gold_file: Path) -> InternalBioNerDataset:
 
         for file in text_files:
             document_id = file.strip(".txt")
-            with open(os.path.join(str(text_dir), file), "r") as text_file:
+            with open(os.path.join(str(text_dir), file), "r", encoding="utf8") as text_file:
                 documents[document_id] = text_file.read().strip()
                 entities_per_document[document_id] = []
 
-        with open(str(gold_file), "r") as gold_reader:
+        with open(str(gold_file), "r", encoding="utf8") as gold_reader:
             for line in gold_reader:
                 if not line:
                     continue
@@ -3358,13 +3358,13 @@ def parse_corpus(
                 if not doc.is_dir():
                     continue
                 try:
-                    with open(doc / "Basedata" / "Basedata.xml", "r") as word_f:
+                    with open(doc / "Basedata" / "Basedata.xml", "r", encoding="utf8") as word_f:
                         word_tree = etree.parse(word_f)
-                    with open(doc / "Markables" / "sentence.xml", "r") as sentence_f:
+                    with open(doc / "Markables" / "sentence.xml", "r", encoding="utf8") as sentence_f:
                         sentence_tree = etree.parse(sentence_f).getroot()
-                    with open(doc / "Markables" / "proteins.xml", "r") as protein_f:
+                    with open(doc / "Markables" / "proteins.xml", "r", encoding="utf8") as protein_f:
                         protein_tree = etree.parse(protein_f).getroot()
-                    with open(doc / "Basedata.uri", "r") as id_f:
+                    with open(doc / "Basedata.uri", "r", encoding="utf8") as id_f:
                         document_id = id_f.read().strip()
                 except FileNotFoundError:
                     # Incomplete article
@@ -3545,14 +3545,14 @@ def parse_corpus(corpus_dir: Path) -> InternalBioNerDataset:
         for doc in Tqdm.tqdm(document_texts, desc="Converting to internal"):
             document_id = doc.name.split(".")[0]
 
-            with open(doc, "r") as f_txt:
+            with open(doc, "r", encoding="utf8") as f_txt:
                 documents[document_id] = f_txt.read()
 
             entities = []
 
             for annotation_dir in annotation_dirs:
                 with open(
-                    annotation_dir / (doc.name + ".annotations.xml"), "r"
+                    annotation_dir / (doc.name + ".annotations.xml"), "r", encoding="utf8"
                 ) as f_ann:
                     ann_tree = etree.parse(f_ann)
                 for annotation in ann_tree.xpath("//annotation"):
@@ -3651,7 +3651,7 @@ def parse_dataset(data_dir: Path) -> InternalBioNerDataset:
 
         for text_file in sorted(text_files):
             document_id = os.path.basename(text_file).split("_")[0]
-            with open(text_file, "r") as file_reader:
+            with open(text_file, "r", encoding="utf8") as file_reader:
                 file_text = file_reader.read().replace("\n", " ")
 
             offset = 0
@@ -3664,7 +3664,7 @@ def parse_dataset(data_dir: Path) -> InternalBioNerDataset:
 
             entities = []
             dirty_file = False
-            with open(text_file[:-4] + ".ann") as file_reader:
+            with open(text_file[:-4] + ".ann", encoding="utf8") as file_reader:
                 for line in file_reader:
                     if line[-1] == "\n":
                         line = line[:-1]
@@ -3797,7 +3797,7 @@ def parse_dataset(text_file: Path, ann_file: Path) -> InternalBioNerDataset:
         documents = {}
         entities_per_document = {}
 
-        with open(str(text_file), "r") as text_file_reader:
+        with open(str(text_file), "r", encoding="utf8") as text_file_reader:
             for line in text_file_reader:
                 line = line.strip()
                 offset = line.find(" ")
@@ -3806,7 +3806,7 @@ def parse_dataset(text_file: Path, ann_file: Path) -> InternalBioNerDataset:
                 documents[document_id] = document_text
                 entities_per_document[document_id] = []
 
-        with open(str(ann_file), "r") as ann_file_reader:
+        with open(str(ann_file), "r", encoding="utf8") as ann_file_reader:
             for line in ann_file_reader:
                 columns = line.strip().split("|")
                 document_id = columns[0]
@@ -3954,7 +3954,7 @@ def parse_input_file(text_file: Path, ann_file: Path) -> InternalBioNerDataset:
         entities_per_document = {}
         document_abstract_length = {}
 
-        with open(str(text_file), "r") as text_reader:
+        with open(str(text_file), "r", encoding="utf8") as text_reader:
             for line in text_reader:
                 if not line:
                     continue
@@ -3967,7 +3967,7 @@ def parse_input_file(text_file: Path, ann_file: Path) -> InternalBioNerDataset:
 
                 entities_per_document[document_id] = []
 
-        with open(str(ann_file), "r") as ann_reader:
+        with open(str(ann_file), "r", encoding="utf8") as ann_reader:
             for line in ann_reader:
                 if not line:
                     continue
@@ -4120,23 +4120,23 @@ def parse_dataset(data_dir: Path, annotator: int) -> InternalBioNerDataset:
         for abstract_id in abstract_ids:
             abstract_id_output = abstract_id + "_A"
             with open(
-                abstract_folder / annotation_dirs[0] / f"{abstract_id}.txt", "r"
+                abstract_folder / annotation_dirs[0] / f"{abstract_id}.txt", "r", encoding="utf8"
             ) as f:
                 documents[abstract_id_output] = f.read()
 
             for annotation_dir in annotation_dirs:
                 with open(
-                    abstract_folder / annotation_dir / f"{abstract_id}.ann", "r"
+                    abstract_folder / annotation_dir / f"{abstract_id}.ann", "r", encoding="utf8"
                 ) as f:
                     entities = CHEBI.get_entities(f)
             entities_per_document[abstract_id_output] = entities
 
         for fulltext_id in fulltext_ids:
             fulltext_id_output = fulltext_id + "_F"
-            with open(fulltext_folder / f"{fulltext_id}.txt", "r") as f:
+            with open(fulltext_folder / f"{fulltext_id}.txt", "r", encoding="utf8") as f:
                 documents[fulltext_id_output] = f.read()
 
-            with open(fulltext_folder / f"{fulltext_id}.ann", "r") as f:
+            with open(fulltext_folder / f"{fulltext_id}.ann", "r", encoding="utf8") as f:
                 entities = CHEBI.get_entities(f)
             entities_per_document[fulltext_id_output] = entities
 
@@ -4291,10 +4291,10 @@ def parse_input_files(input_folder: Path) -> InternalBioNerDataset:
             name = txt_file.with_suffix("").name
             a1_file = txt_file.with_suffix(".a1")
 
-            with txt_file.open() as f:
+            with txt_file.open(encoding="utf8") as f:
                 documents[name] = f.read()
 
-            with a1_file.open() as ann_reader:
+            with a1_file.open(encoding="utf8") as ann_reader:
                 entities = []
 
                 for line in ann_reader:
@@ -4510,7 +4510,7 @@ def parse_input_files(
             sent_offset = 0
             last_offset = 0
 
-            input_file = open(str(input_dir / input_file), "r")
+            input_file = open(str(input_dir / input_file), "r", encoding="utf8")
             for line in input_file.readlines():
                 line = line.strip()
                 if line:
@@ -4577,9 +4577,9 @@ def download_corpora(download_dir: Path):
     @staticmethod
     def convert_and_write(download_folder, data_folder, tag_type):
         data_folder.mkdir(parents=True, exist_ok=True)
-        with (download_folder / "train.tsv").open() as f_in, (
+        with (download_folder / "train.tsv").open(encoding="utf8") as f_in, (
             data_folder / "train.conll"
-        ).open("w") as f_out:
+        ).open("w", encoding="utf8") as f_out:
             for line in f_in:
                 if not line.strip():
                     f_out.write("\n")
@@ -4590,9 +4590,9 @@ def convert_and_write(download_folder, data_folder, tag_type):
                     tag = tag + "-" + tag_type
                 f_out.write(f"{token} {tag}\n")
 
-        with (download_folder / "devel.tsv").open() as f_in, (
+        with (download_folder / "devel.tsv").open(encoding="utf8") as f_in, (
             data_folder / "dev.conll"
-        ).open("w") as f_out:
+        ).open("w", encoding="utf8") as f_out:
             for line in f_in:
                 if not line.strip():
                     f_out.write("\n")
@@ -4602,9 +4602,9 @@ def convert_and_write(download_folder, data_folder, tag_type):
                     tag = tag + "-" + tag_type
                 f_out.write(f"{token} {tag}\n")
 
-        with (download_folder / "test.tsv").open() as f_in, (
+        with (download_folder / "test.tsv").open(encoding="utf8") as f_in, (
             data_folder / "test.conll"
-        ).open("w") as f_out:
+        ).open("w", encoding="utf8") as f_out:
             for line in f_in:
                 if not line.strip():
                     f_out.write("\n")
@@ -5001,7 +5001,7 @@ def prepare_splits(
         splits = {}
         for url in split_urls:
             split_file = cached_path(url, splits_dir)
-            with open(str(split_file), "r") as split_reader:
+            with open(str(split_file), "r", encoding="utf8") as split_reader:
                 splits[url.split(".")[-1]] = [
                     line.strip() for line in split_reader if line.strip()
                 ]
@@ -5053,7 +5053,7 @@ def parse_corpus(corpus_dir: Path) -> InternalBioNerDataset:
         for doc in Tqdm.tqdm(document_texts, desc="Converting to internal"):
             document_id = doc.name.split(".")[0]
 
-            with open(doc, "r") as f_txt:
+            with open(doc, "r", encoding="utf8") as f_txt:
                 documents[document_id] = f_txt.read()
 
             entities = []
@@ -5065,6 +5065,7 @@ def parse_corpus(corpus_dir: Path) -> InternalBioNerDataset:
                     / "knowtator"
                     / (doc.name + ".knowtator.xml"),
                     "r",
+                    encoding="utf8"
                 ) as f_ann:
                     ann_tree = etree.parse(f_ann)
                 for annotation in ann_tree.xpath("//annotation"):
@@ -5442,7 +5443,6 @@ def __init__(self, entity_type: str, sentence_splitter: SentenceSplitter = None)
         def entity_type_predicate(member):
             return f"HUNER_{entity_type}_" in str(member) and inspect.isclass(member)
 
-
         self.huner_corpora_classes = inspect.getmembers(sys.modules[__name__], predicate=entity_type_predicate)
         self.huner_corpora = []
         for name, constructor_func in self.huner_corpora_classes:


Test patch: