Repository: biolab/orange3-text

Issue:
Preprocess Text: crash on no connection
<!--
This is an issue template. Please fill in the relevant details in the
sections below.
-->

##### Text version
<!-- From menu _Options→Add-ons→Orange3-Text_ or code `orangecontrib.text.version.full_version` -->
0.5.2

##### Orange version
<!-- From menu _Help→About→Version_ or code `Orange.version.full_version` -->
3.18.dev

##### Expected behavior
Preprocess Text works without internet connection. Widget provides a warning and connection-dependent blocks are disabled.


##### Actual behavior
Preprocess Text crashes.


##### Steps to reproduce the behavior
Go offline. Open Preprocess Text.


##### Additional info (worksheets, data, screenshots, ...)





Discussion:

Resolving patch:
diff --git a/orangecontrib/text/preprocess/normalize.py b/orangecontrib/text/preprocess/normalize.py
--- a/orangecontrib/text/preprocess/normalize.py
+++ b/orangecontrib/text/preprocess/normalize.py
@@ -3,6 +3,7 @@
 import ufal.udpipe as udpipe
 import serverfiles
 from nltk import stem
+from requests.exceptions import ConnectionError
 from Orange.misc.environ import data_dir
 
 
@@ -115,25 +116,39 @@ def __getitem__(self, language):
         file_name = self._find_file(language_to_name(language))
         return self.localfiles.localpath_download(file_name)
 
+    @property
+    def model_files(self):
+        try:
+            return self.serverfiles.listfiles()
+        except ConnectionError:
+            return self.localfiles.listfiles()
+
     def _find_file(self, language):
         return next(filter(lambda f: file_to_name(f).startswith(language),
-                           map(lambda f: f[0], self.serverfiles.listfiles())))
+                           map(lambda f: f[0], self.model_files)))
 
     @property
     def supported_languages(self):
         self._supported_languages = list(map(lambda f: file_to_language(f[0]),
-                                             self.serverfiles.listfiles()))
+                                             self.model_files))
         return self._supported_languages
 
+    @property
+    def online(self):
+        try:
+            self.serverfiles.listfiles()
+            return True
+        except ConnectionError:
+            return False
+
 
 class UDPipeLemmatizer(BaseNormalizer):
     name = 'UDPipe Lemmatizer'
     str_format = '{self.name} ({self.language})'
-    models = UDPipeModels()
-    supported_languages = models.supported_languages
 
     def __init__(self, language='English'):
         self._language = language
+        self.models = UDPipeModels()
         self.model = None
         self.output_format = udpipe.OutputFormat.newOutputFormat('epe')
         self.use_tokenizer = False
diff --git a/orangecontrib/text/widgets/owpreprocess.py b/orangecontrib/text/widgets/owpreprocess.py
--- a/orangecontrib/text/widgets/owpreprocess.py
+++ b/orangecontrib/text/widgets/owpreprocess.py
@@ -10,6 +10,7 @@
 from Orange.widgets import gui, settings, widget
 from Orange.widgets.widget import OWWidget, Msg, Input, Output
 from orangecontrib.text import preprocess
+from orangecontrib.text.preprocess.normalize import UDPipeModels
 from orangecontrib.text.corpus import Corpus
 from orangecontrib.text.misc import nltk_data_dir
 from orangecontrib.text.tag import StanfordPOSTagger, AveragedPerceptronTagger, \
@@ -278,16 +279,47 @@ def __init__(self, master):
                                               checked=self.udpipe_tokenizer)
         self.udpipe_tokenizer_box.stateChanged.connect(self.change_tokenizer)
         self.method_layout.addWidget(self.udpipe_tokenizer_box, self.UDPIPE, 1)
-        label = gui.label(self, self, 'Language:')
-        label.setAlignment(Qt.AlignRight)
-        self.method_layout.addWidget(label, self.UDPIPE, 2)
-        udpipe_box = widgets.ComboBox(self, 'udpipe_language',
-                               items=preprocess.UDPipeLemmatizer.supported_languages)
-        udpipe_box.currentIndexChanged.connect(self.change_language)
-        self.method_layout.addWidget(udpipe_box, self.UDPIPE, 3)
+        self.udpipe_label = gui.label(self, self, 'Language:')
+        self.udpipe_label.setAlignment(Qt.AlignRight)
+        self.method_layout.addWidget(self.udpipe_label, self.UDPIPE, 2)
+        self.udpipe_models = UDPipeModels()
+        self.create_udpipe_box()
+        self.udpipe_online = self.udpipe_models.online
+        self.on_off_button.stateChanged.connect(self.check_udpipe_online)
+        self.check_udpipe_online()
         self.methods[self.UDPIPE].language = self.udpipe_language
         self.methods[self.UDPIPE].use_tokenizer = self.udpipe_tokenizer
 
+    def create_udpipe_box(self):
+        if not self.udpipe_models.supported_languages:
+            self.group.button(self.UDPIPE).setEnabled(False)
+            self.udpipe_tokenizer_box.setEnabled(False)
+            self.udpipe_label.setEnabled(False)
+            self.udpipe_box = widgets.ComboBox(self, 'udpipe_language', items=[''])
+            self.udpipe_box.setEnabled(False)
+        else:
+            self.group.button(self.UDPIPE).setEnabled(True)
+            self.udpipe_tokenizer_box.setEnabled(True)
+            self.udpipe_label.setEnabled(True)
+            self.udpipe_box = widgets.ComboBox(self, 'udpipe_language',
+                                          items=self.udpipe_models.supported_languages)
+        self.udpipe_box.currentIndexChanged.connect(self.change_language)
+        self.method_layout.addWidget(self.udpipe_box, self.UDPIPE, 3)
+
+    def check_udpipe_online(self):
+        current_state = self.udpipe_models.online
+        if self.udpipe_online != current_state:
+            self.create_udpipe_box()
+            self.udpipe_online = current_state
+
+        self.master.Warning.udpipe_offline.clear()
+        self.master.Warning.udpipe_offline_no_models.clear()
+        if not current_state and self.enabled:
+            if self.udpipe_models.supported_languages:
+                self.master.Warning.udpipe_offline()
+            else:
+                self.master.Warning.udpipe_offline_no_models()
+
     def change_language(self):
         if self.methods[self.SNOWBALL].language != self.snowball_language:
             self.methods[self.SNOWBALL].language = self.snowball_language
@@ -600,6 +632,8 @@ class Error(OWWidget.Error):
 
     class Warning(OWWidget.Warning):
         no_token_left = Msg('No tokens on output! Please, change configuration.')
+        udpipe_offline = Msg('No internet connection! UDPipe now only works with local models.')
+        udpipe_offline_no_models = Msg('No internet connection and no local UDPipe models are available.')
 
     def __init__(self, parent=None):
         super().__init__(parent)


Test patch:
diff --git a/orangecontrib/text/tests/test_preprocess.py b/orangecontrib/text/tests/test_preprocess.py
--- a/orangecontrib/text/tests/test_preprocess.py
+++ b/orangecontrib/text/tests/test_preprocess.py
@@ -4,7 +4,9 @@
 
 import itertools
 import nltk
+from unittest import mock
 from gensim import corpora
+from requests.exceptions import ConnectionError
 import numpy as np
 
 from orangecontrib.text import preprocess
@@ -215,6 +217,23 @@ def test_udpipe_model(self):
         self.assertEqual(model, local_file)
         self.assertTrue(os.path.isfile(local_file))
 
+    def test_udpipe_local_models(self):
+        """Test if UDPipe works offline and uses local models"""
+        models = UDPipeModels()
+        [models.localfiles.remove(f[0]) for f in models.localfiles.listfiles()]
+        _ = models['Slovenian']
+        with mock.patch('serverfiles.ServerFiles.listfiles',
+                        **{'side_effect': ConnectionError()}):
+            self.assertIn('Slovenian', UDPipeModels().supported_languages)
+            self.assertEqual(1, len(UDPipeModels().supported_languages))
+
+    def test_udpipe_offline(self):
+        """Test if UDPipe works offline"""
+        self.assertTrue(UDPipeModels().online)
+        with mock.patch('serverfiles.ServerFiles.listfiles',
+                        **{'side_effect': ConnectionError()}):
+            self.assertFalse(UDPipeModels().online)
+
 
 class FilteringTests(unittest.TestCase):
 
diff --git a/orangecontrib/text/widgets/tests/test_owpreprocess.py b/orangecontrib/text/widgets/tests/test_owpreprocess.py
--- a/orangecontrib/text/widgets/tests/test_owpreprocess.py
+++ b/orangecontrib/text/widgets/tests/test_owpreprocess.py
@@ -1,9 +1,12 @@
+from unittest import mock
+
 from Orange.widgets.tests.base import WidgetTest
+
 from orangecontrib.text.corpus import Corpus
 from orangecontrib.text.widgets.owpreprocess import OWPreprocess
 
 
-class TestConcordanceWidget(WidgetTest):
+class TestPreprocessWidget(WidgetTest):
     def setUp(self):
         self.widget = self.create_widget(OWPreprocess)
         self.corpus = Corpus.from_file('deerwester')
@@ -18,3 +21,27 @@ def test_multiple_instances(self):
         # GH-327
         widget2 = self.create_widget(OWPreprocess)
         widget3 = self.create_widget(OWPreprocess)
+
+    def test_udpipe_offline(self):
+        """Test if preprocess works with offline udpipe"""
+        with mock.patch('orangecontrib.text.preprocess.normalize.UDPipeModels.online',
+                        new_callable=mock.PropertyMock, return_value=False):
+            widget = self.create_widget(OWPreprocess)
+            normalization = widget.stages[2]
+            normalization.on_off_button.click()
+            self.assertTrue(widget.Warning.udpipe_offline.is_shown())
+
+    def test_udpipe_no_models(self):
+        """Test if preprocess disables udpipe if no models"""
+        with mock.patch(
+                'orangecontrib.text.preprocess.normalize.UDPipeModels.online',
+                new_callable=mock.PropertyMock, return_value=False):
+            with mock.patch('orangecontrib.text.preprocess.normalize.UDPipeModels.supported_languages',
+                            new_callable=mock.PropertyMock, return_value=[]):
+                widget = self.create_widget(OWPreprocess)
+                normalization = widget.stages[2]
+                normalization.on_off_button.click()
+                self.assertFalse(normalization.group.button(normalization.UDPIPE)
+                                 .isEnabled())
+                self.assertFalse(normalization.udpipe_tokenizer_box.isEnabled())
+                self.assertTrue(widget.Warning.udpipe_offline_no_models.is_shown())