Repository: biolab/orange3-text

Issue:
WordEnrichment Nans & BoW Features Only
<!--
This is an issue template. Please fill in the relevant details in the
sections below.
-->

##### Text version
<!-- From menu _Options→Add-ons→Orange3-Text_ or code `orangecontrib.text.version.full_version` -->
0.2.4.dev

##### WordEnrichment issues:
* WE takes all features. It should only consider BoW features. If none BoW features present raise an error.
* WE doesn't work if nans in sparse matrix. Probably cannot occur is we use only BoW features.



Discussion:

Resolving patch:
diff --git a/orangecontrib/text/vectorization/bagofwords.py b/orangecontrib/text/vectorization/bagofwords.py
--- a/orangecontrib/text/vectorization/bagofwords.py
+++ b/orangecontrib/text/vectorization/bagofwords.py
@@ -93,7 +93,7 @@ def _transform(self, corpus, source_dict=None):
         shared_cv = BoWPreprocessTransform(corpus.used_preprocessor, self, dic)
         cv = [BoWComputeValue(dic[i], shared_cv) for i in range(len(dic))]
 
-        self.add_features(corpus, X, dic, cv)
+        self.add_features(corpus, X, dic, cv, var_attrs={'bow-feature': True})
         return corpus
 
     def report(self):
diff --git a/orangecontrib/text/vectorization/base.py b/orangecontrib/text/vectorization/base.py
--- a/orangecontrib/text/vectorization/base.py
+++ b/orangecontrib/text/vectorization/base.py
@@ -25,11 +25,20 @@ def report(self):
         raise NotImplementedError
 
     @staticmethod
-    def add_features(corpus, X, dictionary, compute_values=None):
+    def add_features(corpus, X, dictionary, compute_values=None, var_attrs=None):
         order = np.argsort([dictionary[i] for i in range(len(dictionary))])
-        compute_values = np.array(compute_values)[order]
+        if compute_values is not None:
+            compute_values = np.array(compute_values)[order]
+
+        variable_attrs = {
+            'hidden': True,
+            'skip-normalization': True,
+        }
+        if isinstance(var_attrs, dict):
+            variable_attrs.update(var_attrs)
+
         corpus.extend_attributes(X[:, order],
                                  feature_names=(dictionary[i] for i in order),
-                                 var_attrs={'hidden': True, 'skip-normalization': True},
+                                 var_attrs=variable_attrs,
                                  compute_values=compute_values)
         corpus.ngrams_corpus = matutils.Sparse2Corpus(X.T)
diff --git a/orangecontrib/text/widgets/owwordenrichment.py b/orangecontrib/text/widgets/owwordenrichment.py
--- a/orangecontrib/text/widgets/owwordenrichment.py
+++ b/orangecontrib/text/widgets/owwordenrichment.py
@@ -1,12 +1,11 @@
-import math
-
 import numpy as np
 from AnyQt.QtWidgets import QTreeWidget, QTreeView, QTreeWidgetItem
 
-from Orange.data import Table
+from Orange.data import Table, Domain
 from Orange.widgets import gui
 from Orange.widgets.settings import Setting
 from Orange.widgets.widget import OWWidget, Msg
+from orangecontrib.text import Corpus
 from orangecontrib.text.util import np_sp_sum
 from orangecontrib.text.stats import false_discovery_rate, hypergeom_p_values
 
@@ -24,6 +23,7 @@ class OWWordEnrichment(OWWidget):
     want_main_area = True
 
     class Error(OWWidget.Error):
+        no_bow_features = Msg('No bag-of-words features!')
         no_words_overlap = Msg('No words overlap!')
         empty_selection = Msg('Selected data is empty!')
         all_selected = Msg('All examples can not be selected!')
@@ -99,6 +99,15 @@ def set_data_selected(self, data=None):
     def handleNewSignals(self):
         self.check_data()
 
+    def get_bow_domain(self):
+        domain = self.data.domain
+        return Domain(
+            attributes=[a for a in domain.attributes
+                        if a.attributes.get('bow-feature', False)],
+            class_vars=domain.class_vars,
+            metas=domain.metas,
+            source=domain)
+
     def check_data(self):
         self.Error.clear()
         if isinstance(self.data, Table) and \
@@ -108,11 +117,16 @@ def check_data(self):
                 self.clear()
                 return
 
-            self.selected_data_transformed = Table.from_table(
-                self.data.domain, self.selected_data)
+            # keep only BoW features
+            bow_domain = self.get_bow_domain()
+            if len(bow_domain.attributes) == 0:
+                self.Error.no_bow_features()
+                self.clear()
+                return
+            self.data = Corpus.from_table(bow_domain, self.data)
+            self.selected_data_transformed = Corpus.from_table(bow_domain, self.selected_data)
 
-            sum_X = np_sp_sum(self.selected_data_transformed.X)
-            if sum_X == 0 or math.isnan(sum_X):
+            if np_sp_sum(self.selected_data_transformed.X) == 0:
                 self.Error.no_words_overlap()
                 self.clear()
             elif len(self.data) == len(self.selected_data):


Test patch:
diff --git a/orangecontrib/text/tests/test_vectorization_base.py b/orangecontrib/text/tests/test_vectorization_base.py
new file mode 100644
--- /dev/null
+++ b/orangecontrib/text/tests/test_vectorization_base.py
@@ -0,0 +1,34 @@
+import unittest
+
+import numpy as np
+import scipy.sparse as sp
+
+from orangecontrib.text.corpus import Corpus
+from orangecontrib.text.vectorization.base import BaseVectorizer
+
+
+class BaseVectorizationTest(unittest.TestCase):
+    def test_variable_attributes(self):
+        c1 = Corpus.from_file('deerwester')
+        c2 = Corpus.from_file('deerwester')
+        X = np.array([list(range(4)) for _ in range(len(c1))])
+        X = sp.csr_matrix(X)
+
+        dictionary = {
+            0: 'd',
+            1: 'c',
+            2: 'b',
+            3: 'a',
+        }
+
+        BaseVectorizer.add_features(c1, X, dictionary,
+                                    compute_values=None, var_attrs=None)
+        BaseVectorizer.add_features(c2, X, dictionary,
+                                    compute_values=None, var_attrs={'foo': 1})
+
+        n_attrs_before = len(c1.domain.attributes[0].attributes)
+        n_attrs_after = len(c2.domain.attributes[0].attributes)
+        self.assertTrue(n_attrs_after - n_attrs_before, 1)
+
+        for a in c2.domain.attributes:
+            self.assertIn('foo', a.attributes)