Repository: flairNLP/flair

Issue:
Extend ColumnCorpus with option to define "space after"
The Universal Dependencies column format has a way of marking whether or not a whitespace follows a token (our `UniversalDependenciesCorpus` is able to recognize this). However, a similar mechanism does not currently exist for other column-formatted datasets which we read with the `ColumnCorpus` object. So far this was not necessary as all datasets we've encountered, such as CoNLL-03, do not include this information. 

In the context of #1513, we've now come across a use case in which we need to know whether a whitespace follows a token in column format. This issue tracks the extension of `ColumnCorpus` to allow us to include this information.


Discussion:

Resolving patch:
diff --git a/flair/datasets/sequence_labeling.py b/flair/datasets/sequence_labeling.py
--- a/flair/datasets/sequence_labeling.py
+++ b/flair/datasets/sequence_labeling.py
@@ -93,6 +93,10 @@ def __init__(
 
 
 class ColumnDataset(FlairDataset):
+
+    # special key for space after
+    SPACE_AFTER_KEY = "space-after"
+
     def __init__(
             self,
             path_to_column_file: Path,
@@ -166,7 +170,6 @@ def __init__(
 
                     if sentence_started:
 
-                        sentence.infer_space_after()
                         if self.in_memory:
                             if self.tag_to_bioes is not None:
                                 sentence.convert_tag_scheme(
@@ -181,31 +184,36 @@ def __init__(
                     sentence_started = False
 
                 elif self.in_memory:
-                    fields: List[str] = re.split(self.column_delimiter, line)
-                    token = Token(fields[self.text_column])
-                    for column in column_name_map:
-                        if len(fields) > column:
-                            if column != self.text_column:
-                                token.add_label(
-                                    self.column_name_map[column], fields[column]
-                                )
-
+                    token = self._parse_token(line)
                     if not line.isspace():
                         sentence.add_token(token)
                         sentence_started = True
+
                 elif not line.isspace():
                     sentence_started = True
 
                 line = f.readline()
 
         if sentence_started:
-            sentence.infer_space_after()
             if self.in_memory:
                 self.sentences.append(sentence)
             else:
                 self.indices.append(position)
             self.total_sentence_count += 1
 
+    def _parse_token(self, line: str) -> Token:
+        fields: List[str] = re.split(self.column_delimiter, line)
+        token = Token(fields[self.text_column])
+        for column in self.column_name_map:
+            if len(fields) > column:
+                if column != self.text_column and self.column_name_map[column] != self.SPACE_AFTER_KEY:
+                    token.add_label(
+                        self.column_name_map[column], fields[column]
+                    )
+                if self.column_name_map[column] == self.SPACE_AFTER_KEY and fields[column] == '-':
+                        token.whitespace_after = False
+        return token
+
     def __line_completes_sentence(self, line: str) -> bool:
         sentence_completed = line.isspace()
         if self.document_separator_token:
@@ -241,7 +249,6 @@ def __getitem__(self, index: int = 0) -> Sentence:
 
                     if self.__line_completes_sentence(line):
                         if len(sentence) > 0:
-                            sentence.infer_space_after()
                             if self.tag_to_bioes is not None:
                                 sentence.convert_tag_scheme(
                                     tag_type=self.tag_to_bioes, target_scheme="iobes"
@@ -249,15 +256,7 @@ def __getitem__(self, index: int = 0) -> Sentence:
                             return sentence
 
                     else:
-                        fields: List[str] = re.split(self.column_delimiter, line)
-                        token = Token(fields[self.text_column])
-                        for column in self.column_name_map:
-                            if len(fields) > column:
-                                if column != self.text_column:
-                                    token.add_label(
-                                        self.column_name_map[column], fields[column]
-                                    )
-
+                        token = self._parse_token(line)
                         if not line.isspace():
                             sentence.add_token(token)
 


Test patch:
diff --git a/tests/resources/tasks/column_corpus_options/eng.testa b/tests/resources/tasks/column_corpus_options/eng.testa
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/column_corpus_options/eng.testa
@@ -0,0 +1,4 @@
+WORD	TAG
+This	O
+is	O
+Coca Cola	O
diff --git a/tests/resources/tasks/column_corpus_options/eng.testb b/tests/resources/tasks/column_corpus_options/eng.testb
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/column_corpus_options/eng.testb
@@ -0,0 +1,4 @@
+WORD	TAG
+This	O
+is	O
+New York	O
diff --git a/tests/resources/tasks/column_corpus_options/eng.train b/tests/resources/tasks/column_corpus_options/eng.train
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/column_corpus_options/eng.train
@@ -0,0 +1,4 @@
+WORD	TAG
+This	O
+is	O
+New Berlin	LOC
diff --git a/tests/resources/tasks/column_with_whitespaces/eng.testa b/tests/resources/tasks/column_with_whitespaces/eng.testa
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/column_with_whitespaces/eng.testa
@@ -0,0 +1,8 @@
+It O +
+is O +
+a O +
+French B-LOC -
+- O -
+speaking O +
+town O -
+. O +
\ No newline at end of file
diff --git a/tests/resources/tasks/column_with_whitespaces/eng.testb b/tests/resources/tasks/column_with_whitespaces/eng.testb
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/column_with_whitespaces/eng.testb
@@ -0,0 +1,8 @@
+It O +
+is O +
+a O +
+US B-LOC -
+- O -
+based O +
+company O -
+. O +
\ No newline at end of file
diff --git a/tests/resources/tasks/column_with_whitespaces/eng.train b/tests/resources/tasks/column_with_whitespaces/eng.train
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/column_with_whitespaces/eng.train
@@ -0,0 +1,8 @@
+It O +
+is O +
+a O +
+German B-LOC -
+- O -
+owned O +
+firm O -
+. O +
\ No newline at end of file
diff --git a/tests/test_data.py b/tests/test_data.py
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -104,6 +104,7 @@ def test_sentence_to_real_string(tasks_base_path):
     corpus = flair.datasets.GERMEVAL_14(base_path=tasks_base_path)
 
     sentence = corpus.train[0]
+    sentence.infer_space_after()
     assert (
         'Schartau sagte dem " Tagesspiegel " vom Freitag , Fischer sei " in einer Weise aufgetreten , die alles andere als überzeugend war " .'
         == sentence.to_tokenized_string()
@@ -114,6 +115,7 @@ def test_sentence_to_real_string(tasks_base_path):
     )
 
     sentence = corpus.train[1]
+    sentence.infer_space_after()
     assert (
         "Firmengründer Wolf Peter Bree arbeitete Anfang der siebziger Jahre als Möbelvertreter , als er einen fliegenden Händler aus dem Libanon traf ."
         == sentence.to_tokenized_string()
diff --git a/tests/test_data_fetchers.py b/tests/test_datasets.py
similarity index 79%
rename from tests/test_data_fetchers.py
rename to tests/test_datasets.py
--- a/tests/test_data_fetchers.py
+++ b/tests/test_datasets.py
@@ -70,6 +70,35 @@ def test_load_sequence_labeling_data(tasks_base_path):
     assert len(corpus.test) == 1
 
 
+def test_load_sequence_labeling_whitespace_after(tasks_base_path):
+    # get training, test and dev data
+    corpus = flair.datasets.ColumnCorpus(
+        tasks_base_path / "column_with_whitespaces", column_format={0: 'text', 1: 'ner', 2: 'space-after'}
+    )
+
+    assert len(corpus.train) == 1
+    assert len(corpus.dev) == 1
+    assert len(corpus.test) == 1
+
+    assert corpus.train[0].to_tokenized_string() == "It is a German - owned firm ."
+    assert corpus.train[0].to_plain_string() == "It is a German-owned firm."
+
+
+def test_load_column_corpus_options(tasks_base_path):
+    # get training, test and dev data
+    corpus = flair.datasets.ColumnCorpus(
+        tasks_base_path / "column_corpus_options",
+        column_format={0: 'text', 1: 'ner'},
+        column_delimiter='\t',
+        skip_first_line=True,
+    )
+
+    assert len(corpus.train) == 1
+    assert len(corpus.dev) == 1
+    assert len(corpus.test) == 1
+
+    assert corpus.train[0].to_tokenized_string() == "This is New Berlin"
+
 def test_load_germeval_data(tasks_base_path):
     # get training, test and dev data
     corpus = flair.datasets.GERMEVAL_14(tasks_base_path)