Repository: biolab/orange3

Issue:
OWSql: Support PostgreSQL servers without extensions
Orange requires two extensions to be installed on PostgreSQL server it connects to; quantile (used for discretization) and tsm_system_time (used for efficient sampling). Instructions on how to install these extensions on the server are available on wiki: https://github.com/biolab/orange3/wiki/Installation-of-PostgreSQL-extensions

Many users, however, connect to servers that do not have these extensions installed and sometimes do not have the necessary permissions to install them. For a list of reports, see gh-2083

As extensions are only required for online work with the data, owsql widget could be modified to force offline mode (enable Download data to local memory checkbox and make it non-editable) and show a warning instead of an error. This way, users can still access the data, they just need to load the whole table into memory to perform the analysis.


Discussion:
> they just need to load the whole table into memory

Or optionally a reasonably large while still manageable [sample](https://www.periscopedata.com/blog/how-to-sample-rows-in-sql-273x-faster).

Resolving patch:
diff --git a/Orange/data/sql/backend/postgres.py b/Orange/data/sql/backend/postgres.py
--- a/Orange/data/sql/backend/postgres.py
+++ b/Orange/data/sql/backend/postgres.py
@@ -4,7 +4,7 @@
 from contextlib import contextmanager
 from time import time
 
-from psycopg2 import Error, OperationalError
+from psycopg2 import Error
 from psycopg2.pool import ThreadedConnectionPool
 
 from Orange.data import ContinuousVariable, DiscreteVariable, StringVariable, TimeVariable
@@ -29,6 +29,7 @@ def __init__(self, connection_params):
         if self.connection_pool is None:
             self._create_connection_pool()
 
+        self.missing_extension = []
         if self.auto_create_extensions:
             self._create_extensions()
 
@@ -45,8 +46,9 @@ def _create_extensions(self):
                 query = "CREATE EXTENSION IF NOT EXISTS {}".format(ext)
                 with self.execute_sql_query(query):
                     pass
-            except OperationalError:
+            except BackendError:
                 warnings.warn("Database is missing extension {}".format(ext))
+                self.missing_extension.append(ext)
 
     def create_sql_query(self, table_name, fields, filters=(),
                          group_by=None, order_by=None,
diff --git a/Orange/widgets/data/owsql.py b/Orange/widgets/data/owsql.py
--- a/Orange/widgets/data/owsql.py
+++ b/Orange/widgets/data/owsql.py
@@ -21,6 +21,10 @@
 MAX_DL_LIMIT = 1000000
 
 
+def is_postgres(backend):
+    return getattr(backend, 'display_name', '') == "PostgreSQL"
+
+
 class TableModel(PyListModel):
     def data(self, index, role=Qt.DisplayRole):
         row = index.row()
@@ -74,7 +78,9 @@ class Information(OWWidget.Information):
     class Error(OWWidget.Error):
         connection = Msg("{}")
         no_backends = Msg("Please install a backend to use this widget")
-        missing_extension = Msg("Database is missing extension{}: {}")
+
+    class Warning(OWWidget.Warning):
+        missing_extension = Msg("Database is missing extensions: {}")
 
     def __init__(self):
         super().__init__()
@@ -135,7 +141,6 @@ def __init__(self):
         self.tablecombo.setModel(self.tables)
         self.tablecombo.setToolTip('table')
         tables.layout().addWidget(self.tablecombo)
-        self.connect()
 
         index = self.tablecombo.findText(str(self.table))
         if index != -1:
@@ -170,12 +175,14 @@ def __init__(self):
                      "Auto-discover categorical variables",
                      callback=self.open_table)
 
-        gui.checkBox(box, self, "download",
-                     "Download data to local memory",
-                     callback=self.open_table)
+        self.downloadcb = gui.checkBox(box, self, "download",
+                                       "Download data to local memory",
+                                       callback=self.open_table)
 
         gui.rubber(self.buttonsArea)
 
+        self.connect()
+
         QTimer.singleShot(0, self.select_table)
 
     def _load_credentials(self):
@@ -223,6 +230,8 @@ def connect(self):
         self.database, _, self.schema = self.databasetext.text().partition('/')
         self.username = self.usernametext.text() or None
         self.password = self.passwordtext.text() or None
+        self.Warning.missing_extension.clear()
+        self.downloadcb.setEnabled(True)
         try:
             if self.backendcombo.currentIndex() < 0:
                 return
@@ -235,6 +244,12 @@ def connect(self):
                 password=self.password
             ))
             self.Error.connection.clear()
+            if getattr(self.backend, 'missing_extension', False):
+                self.Warning.missing_extension(
+                    ", ".join(self.backend.missing_extension))
+                self.download = True
+                self.downloadcb.setEnabled(False)
+
             self._save_credentials()
             self.database_desc = OrderedDict((
                 ("Host", self.host), ("Port", self.port),
@@ -249,7 +264,6 @@ def connect(self):
 
     def refresh_tables(self):
         self.tables.clear()
-        self.Error.missing_extension.clear()
         if self.backend is None:
             self.data_desc_table = None
             return
@@ -344,12 +358,14 @@ def get_table(self):
                             "Do you want to auto discover attributes?")
             confirm.addButton("Yes", QMessageBox.YesRole)
             no_button = confirm.addButton("No", QMessageBox.NoRole)
-            sample_button = confirm.addButton("Yes, on a sample",
-                                              QMessageBox.YesRole)
+            if is_postgres(self.backend):
+                sample_button = confirm.addButton("Yes, on a sample",
+                                                  QMessageBox.YesRole)
             confirm.exec()
             if confirm.clickedButton() == no_button:
                 self.guess_values = False
-            elif confirm.clickedButton() == sample_button:
+            elif is_postgres(self.backend) and \
+                    confirm.clickedButton() == sample_button:
                 sample = True
 
         self.Information.clear()
@@ -365,20 +381,38 @@ def get_table(self):
             table.domain = domain
 
         if self.download:
-            if table.approx_len() > MAX_DL_LIMIT:
-                QMessageBox.warning(
-                    self, 'Warning', "Data is too big to download.\n"
-                    "Consider using the Data Sampler widget to download "
-                    "a sample instead.")
-                self.download = False
-            elif table.approx_len() > AUTO_DL_LIMIT:
-                confirm = QMessageBox.question(
-                    self, 'Question', "Data appears to be big. Do you really "
-                                      "want to download it to local memory?",
-                    QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
-                if confirm == QMessageBox.No:
-                    self.download = False
-        if self.download:
+            if table.approx_len() > AUTO_DL_LIMIT:
+                if is_postgres(self.backend):
+                    confirm = QMessageBox(self)
+                    confirm.setIcon(QMessageBox.Warning)
+                    confirm.setText("Data appears to be big. Do you really "
+                                    "want to download it to local memory?")
+
+                    if table.approx_len() <= MAX_DL_LIMIT:
+                        confirm.addButton("Yes", QMessageBox.YesRole)
+                    no_button = confirm.addButton("No", QMessageBox.NoRole)
+                    sample_button = confirm.addButton("Yes, a sample",
+                                                      QMessageBox.YesRole)
+                    confirm.exec()
+                    if confirm.clickedButton() == no_button:
+                        return
+                    elif confirm.clickedButton() == sample_button:
+                        table = table.sample_percentage(
+                            AUTO_DL_LIMIT / table.approx_len() * 100)
+                else:
+                    if table.approx_len() > MAX_DL_LIMIT:
+                        QMessageBox.warning(
+                            self, 'Warning', "Data is too big to download.\n")
+                        return
+                    else:
+                        confirm = QMessageBox.question(
+                            self, 'Question',
+                            "Data appears to be big. Do you really "
+                            "want to download it to local memory?",
+                            QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
+                        if confirm == QMessageBox.No:
+                            return
+
             table.download_data(MAX_DL_LIMIT)
             table = Table(table)
 


Test patch:
diff --git a/Orange/widgets/data/tests/test_owsql.py b/Orange/widgets/data/tests/test_owsql.py
new file mode 100644
--- /dev/null
+++ b/Orange/widgets/data/tests/test_owsql.py
@@ -0,0 +1,72 @@
+# Test methods with long descriptive names can omit docstrings
+# pylint: disable=missing-docstring
+
+import unittest
+
+from Orange.data import Table
+from Orange.widgets.data.owsql import OWSql
+from Orange.widgets.tests.base import WidgetTest
+from Orange.tests.sql.base import create_iris, parse_uri, sql_test
+
+
+@sql_test
+class TestOWSql(WidgetTest):
+    def setUp(self):
+        self.widget = self.create_widget(OWSql)
+        params, _ = create_iris()
+        self.params = parse_uri(params)
+        self.iris = Table("iris")
+
+    def test_connection(self):
+        """Test if a connection to the database can be established"""
+        self.set_connection_params()
+        self.widget.connect()
+
+        self.assertFalse(self.widget.Error.connection.is_shown())
+        self.assertIsNotNone(self.widget.database_desc)
+        tables = ["Select a table", "Custom SQL"]
+        self.assertTrue(set(self.widget.tables).issuperset(set(tables)))
+
+    def test_output_iris(self):
+        """Test if iris data can be fetched from database"""
+        self.assertIsNone(self.get_output(self.widget.Outputs.data))
+
+        self.set_connection_params()
+        self.widget.connect()
+
+        idx = list(map(str, self.widget.tables)).index("iris")
+        self.widget.tablecombo.setCurrentIndex(idx)
+        self.widget.select_table()
+
+        output = self.get_output(self.widget.Outputs.data)
+        self.assertIsNotNone(output)
+        self.assertEqual(len(output), len(self.iris))
+        iris_domain = set(map(str, self.iris.domain.attributes))
+        output_domain = set(map(str, output.domain.attributes))
+        self.assertTrue(output_domain.issuperset(iris_domain))
+
+    def test_missing_extension(self):
+        """Test for correctly handled missing backend extension"""
+        self.set_connection_params()
+        self.widget.backends[0] = unittest.mock.Mock()
+        self.widget.backends[0]().missing_extension = ["missing extension"]
+        self.widget.backends[0]().list_tables = lambda x: []
+        self.widget.connect()
+
+        self.assertTrue(self.widget.Warning.missing_extension.is_shown())
+        self.assertTrue(self.widget.download)
+        self.assertFalse(self.widget.downloadcb.isEnabled())
+
+    def set_connection_params(self):
+        """Set database connection parameters on widget"""
+        port = ''
+        if self.params['port'] is not None:
+            port += ':' + str(self.params['port'])
+        self.widget.servertext.setText(self.params['host'] + port)
+        self.widget.databasetext.setText(self.params['database'])
+        self.widget.usernametext.setText(self.params['user'])
+        self.widget.passwordtext.setText(self.params['password'])
+
+
+if __name__ == "__main__":
+    unittest.main()