Repository: biolab/orange3-text

Issue:
Preprocess Text: consider pos tags when filtering tokens
**Describe the bug**
If pos tags exists before filtering, final pos tags don't match the tokens. This is because tokens are filtered independently from pos tags and only zipped at the end.

**To Reproduce**
Steps to reproduce the behavior:
1. Corpus
2. Preprocess Text (have POS tagger before filtering on stopwords)
3. Check the result in Corpus Viewer

**Expected behavior**
If pos tags exist, they are filtered together with tokens.

**Orange version:**
3.30.dev

**Text add-on version:**
1.6.dev

**Screenshots**
<img width="509" alt="Screen Shot 2021-07-21 at 15 13 50" src="https://user-images.githubusercontent.com/12524972/126494455-90d78da1-d0d3-4a02-be65-e176268c4c73.png">

**Operating system:**
OSX

**Additional context**
/



Discussion:

Resolving patch:
diff --git a/orangecontrib/text/preprocess/filter.py b/orangecontrib/text/preprocess/filter.py
--- a/orangecontrib/text/preprocess/filter.py
+++ b/orangecontrib/text/preprocess/filter.py
@@ -1,3 +1,4 @@
+from itertools import compress
 from typing import List, Callable
 import os
 import re
@@ -25,14 +26,27 @@ def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:
         corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))
         return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))
 
-    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
+    def _filter_tokens(self, corpus: Corpus, callback: Callable,
+                       dictionary=None) -> Corpus:
         callback(0, "Filtering...")
-        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]
-        corpus.store_tokens(tokens)
+        filtered_tokens = []
+        filtered_tags = []
+        for i, tokens in enumerate(corpus.tokens):
+            filter_map = self._preprocess(tokens)
+            filtered_tokens.append(list(compress(tokens, filter_map)))
+            if corpus.pos_tags is not None:
+                filtered_tags.append(list(compress(corpus.pos_tags[i],
+                                                   filter_map)))
+        if dictionary is None:
+            corpus.store_tokens(filtered_tokens)
+        else:
+            corpus.store_tokens(filtered_tokens, dictionary)
+        if filtered_tags:
+            corpus.pos_tags = np.array(filtered_tags, dtype=object)
         return corpus
 
     def _preprocess(self, tokens: List) -> List:
-        return list(filter(self._check, tokens))
+        return [self._check(token) for token in tokens]
 
     def _check(self, token: str) -> bool:
         raise NotImplementedError
@@ -141,10 +155,10 @@ def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:
     def _fit(self, corpus: Corpus):
         raise NotImplemented
 
-    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
-        callback(0, "Filtering...")
-        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]
-        corpus.store_tokens(tokens, self._dictionary)
+    def _filter_tokens(self, corpus: Corpus, callback: Callable,
+                       dictionary=None) -> Corpus:
+        corpus = super()._filter_tokens(corpus, callback,
+                                        dictionary=self._dictionary)
         return corpus
 
     def _check(self, token):


Test patch:
diff --git a/orangecontrib/text/tests/test_preprocess.py b/orangecontrib/text/tests/test_preprocess.py
--- a/orangecontrib/text/tests/test_preprocess.py
+++ b/orangecontrib/text/tests/test_preprocess.py
@@ -115,6 +115,20 @@ def test_retain_ids(self):
             corpus = pp(corpus)
         self.assertTrue((corpus.ids == self.corpus.ids).all())
 
+    def test_filter_pos_tags(self):
+        pp_list = [preprocess.LowercaseTransformer(),
+                   preprocess.WordPunctTokenizer(),
+                   tag.AveragedPerceptronTagger(),
+                   preprocess.StopwordsFilter()]
+        corpus = self.corpus
+        corpus.metas[0, 0] = "This is the most beautiful day in the world"
+        for pp in pp_list:
+            corpus = pp(corpus)
+        self.assertEqual(len(corpus.tokens), len(corpus.pos_tags))
+        self.assertEqual(len(corpus.tokens[0]), len(corpus.pos_tags[0]))
+        self.assertEqual(corpus.tokens[0], ["beautiful", "day", "world"])
+        self.assertEqual(corpus.pos_tags[0], ["JJ", "NN", "NN"])
+
 
 class TransformationTests(unittest.TestCase):
     def setUp(self):
@@ -331,8 +345,11 @@ def _check(self, token):
                 return not token.isdigit()
 
         df = DigitsFilter()
-        self.assertEqual(df._preprocess([]), [])
-        self.assertEqual(df._preprocess(['a', '1']), ['a'])
+        filtered = list(itertools.compress([], df._preprocess([])))
+        self.assertEqual(filtered, [])
+        filtered = list(itertools.compress(['a', '1'],
+                                           df._preprocess(['a', '1'])))
+        self.assertEqual(filtered, ['a'])
 
     def test_stopwords(self):
         f = preprocess.StopwordsFilter('english')