Repository: biolab/orange3

Issue:
OWGoogleSheets: new data source
@BlazZupan and @marinkaz might also want to review this. Should eventually work with https://github.com/biolab/orange3/pull/524.



Discussion:
I tested the functionality on a few google sheets. Somehow the first two data rows are always lost and never appear in the downloaded data. @kernc, can you take a look into the issue?

Well, that's improperly parsed TSV, something fixed in #524. Do have a look at that if you will. :smile_cat: 

Google just "refreshed" [their logo and the 'G'](http://googleblog.blogspot.com/2015/09/google-update.html). Anything about it?

With the newest changes, this is now blocked on #524.


Resolving patch:
diff --git a/Orange/data/table.py b/Orange/data/table.py
--- a/Orange/data/table.py
+++ b/Orange/data/table.py
@@ -1,4 +1,5 @@
 import os
+import re
 import zlib
 from collections import MutableSequence, Iterable, Sequence, Sized
 from itertools import chain
@@ -7,9 +8,10 @@
 from functools import reduce
 from warnings import warn
 from threading import Lock
-import tempfile
-import urllib.parse
-import urllib.request
+from tempfile import NamedTemporaryFile
+from urllib.parse import urlparse, unquote as urlunquote
+from urllib.request import urlopen
+from urllib.error import URLError
 
 import bottlechest as bn
 from scipy import sparse as sp
@@ -523,13 +525,53 @@ def from_file(filename, wrapper=None):
 
     @classmethod
     def from_url(cls, url):
-        name = os.path.basename(urllib.parse.urlparse(url)[2])
-        f = tempfile.NamedTemporaryFile(suffix=name, delete=False)
-        fname = f.name
-        f.close()
-        urllib.request.urlretrieve(url, fname)
-        data = cls.from_file(f.name)
-        os.remove(fname)
+        # Resolve (potential) redirects to a final URL
+        response = urlopen(url, timeout=10)
+        url = response.url
+        response.close()
+
+        def url_googlesheets(url):
+            match = re.match(r'(?:https?://)?(?:www\.)?'
+                             'docs\.google\.com/spreadsheets/d/'
+                             '(?P<workbook_id>[-\w_]+)'
+                             '(?:/.*?gid=(?P<sheet_id>\d+).*|.*)?',
+                             url, re.IGNORECASE)
+            try:
+                workbook, sheet = match.group('workbook_id'), match.group('sheet_id')
+                if not workbook:
+                    raise ValueError
+            except (AttributeError, ValueError):
+                raise ValueError
+            url = 'https://docs.google.com/spreadsheets/d/{}/export?format=tsv'.format(workbook)
+            if sheet:
+                url += '&gid=' + sheet
+            return url
+
+        URL_TRIMMERS = (
+            url_googlesheets,
+        )
+        for trim in URL_TRIMMERS:
+            try: url = trim(url)
+            except ValueError: continue
+            else: break
+
+        name = urlparse(url)[2].replace('/', '_')
+
+        def suggested_filename(content_disposition):
+            # See https://tools.ietf.org/html/rfc6266#section-4.1
+            matches = re.findall(r"filename\*?=(?:\"|.{0,10}?'[^']*')([^\"]+)",
+                                 content_disposition or '')
+            return urlunquote(matches[-1]) if matches else ''
+
+        with urlopen(url, timeout=10) as response:
+            name = suggested_filename(response.headers['content-disposition']) or name
+            with NamedTemporaryFile(suffix=name) as f:
+                f.write(response.read())
+                f.flush()
+                data = cls.from_file(f.name)
+        # Override name set in from_file() to avoid holding the temp prefix
+        data.name = os.path.splitext(name)[0]
+        data.origin = url
         return data
 
     # Helper function for __setitem__ and insert:
diff --git a/Orange/widgets/data/owfile.py b/Orange/widgets/data/owfile.py
--- a/Orange/widgets/data/owfile.py
+++ b/Orange/widgets/data/owfile.py
@@ -1,7 +1,6 @@
 import os
 import sys
 import urllib
-from collections import namedtuple
 
 from PyQt4 import QtGui
 from Orange.widgets import widget, gui
@@ -21,22 +20,27 @@ def add_origin(examples, filename):
             var.attributes["origin"] = dir_name
 
 
-RecentPath = namedtuple(
-    "RecentPath",
-    ["abspath",   #: str # absolute path
-     "prefix",    #: Option[str]  # BASEDIR | SAMPLE-DATASETS | ...
-     "relpath"]   #: Option[str]  # path relative to `prefix`
-)
+class RecentPath:
+    abspath = ''
+    prefix = None   #: Option[str]  # BASEDIR | SAMPLE-DATASETS | ...
+    relpath = ''  #: Option[str]  # path relative to `prefix`
+    title = ''    #: Option[str]  # title of filename (e.g. from URL)
 
-
-class RecentPath(RecentPath):
-    def __new__(cls, abspath, prefix, relpath):
+    def __init__(self, abspath, prefix, relpath, title=''):
         if os.name == "nt":
             # always use a cross-platform pathname component separator
             abspath = abspath.replace(os.path.sep, "/")
             if relpath is not None:
                 relpath = relpath.replace(os.path.sep, "/")
-        return super(RecentPath, cls).__new__(cls, abspath, prefix, relpath)
+        self.abspath = abspath
+        self.prefix = prefix
+        self.relpath = relpath
+        self.title = title
+
+    def __eq__(self, other):
+        return (self.abspath == other.abspath or
+                self.prefix == other.prefix and
+                self.relpath == other.relpath)
 
     @staticmethod
     def create(path, searchpaths):
@@ -118,7 +122,7 @@ def resolve(self, searchpaths):
     @property
     def value(self):
         if self.prefix == "url-datasets":
-            return self.abspath
+            return '{} ({})'.format(self.title, self.abspath)
         return os.path.basename(self.abspath)
 
     @property
@@ -133,11 +137,6 @@ def dirname(self):
         return os.path.dirname(self.abspath)
 
 
-class RecentPathDelegate(QtGui.QStyledItemDelegate):
-    def displayText(self, value, locale):
-        return os.path.basename(value)
-
-
 class OWFile(widget.OWWidget):
     name = "File"
     id = "orange.widgets.data.file"
@@ -180,10 +179,9 @@ def __init__(self):
         vbox = gui.widgetBox(self.controlArea, "Data File / URL",
                              addSpace=True)
         box = gui.widgetBox(vbox, orientation=0)
-        self.file_combo = QtGui.QComboBox(box)
+        self.file_combo = QtGui.QComboBox(box, sizeAdjustPolicy=QtGui.QComboBox.AdjustToContents)
         self.file_combo.setMinimumWidth(300)
         self.file_combo.setEditable(True)
-        self.file_combo.setItemDelegate(RecentPathDelegate())
         self.file_combo.lineEdit().setStyleSheet("padding-left: 1px;")
         box.layout().addWidget(self.file_combo)
         self.file_combo.activated[int].connect(self.select_file)
@@ -206,9 +204,9 @@ def __init__(self):
                      "represent different variables")
 
         box = gui.widgetBox(self.controlArea, "Info", addSpace=True)
-        self.infoa = gui.widgetLabel(box, 'No data loaded.')
-        self.infob = gui.widgetLabel(box, ' ')
+        self.info = gui.widgetLabel(box, 'No data loaded.')
         self.warnings = gui.widgetLabel(box, ' ')
+        gui.rubber(box)
         #Set word wrap, so long warnings won't expand the widget
         self.warnings.setWordWrap(True)
         self.warnings.setSizePolicy(
@@ -262,7 +260,7 @@ def reload(self):
         if self.recent_paths:
             basename = self.file_combo.currentText()
             if (basename == self.recent_paths[0].relpath or
-                basename == os.path.basename(self.recent_paths[0].abspath)):
+                basename == self.recent_paths[0].value):
                 return self.open_file(self.recent_paths[0].abspath)
         self.select_file(len(self.recent_paths) + 1)
 
@@ -292,8 +290,8 @@ def select_file(self, n):
                     return
 
         if len(self.recent_paths) > 0:
-            self.set_file_list()
             self.open_file(self.recent_paths[0].abspath)
+            self.set_file_list()
 
     def browse_file(self, in_demos=0):
         if in_demos:
@@ -372,14 +370,15 @@ def open_file(self, fn):
                                  .format(basename))
         if fn == "(none)":
             self.send("Data", None)
-            self.infoa.setText("No data loaded")
-            self.infob.setText("")
+            self.info.setText("No data loaded")
             self.warnings.setText("")
             return
 
         self.loaded_file = ""
 
         data = None
+        progress = gui.ProgressBar(self, 3)
+        progress.advance()
         try:
             # TODO handle self.new_variables
             data = Table(fn)
@@ -399,37 +398,43 @@ def open_file(self, fn):
                             self.recent_paths[ind].abspath == fn_original:
                 del self.recent_paths[ind]
             self.error(err_value)
-            self.infoa.setText('Data was not loaded due to an error.')
-            self.infob.setText('Error:')
+            self.info.setText('Data was not loaded due to an error.\nError:')
             self.warnings.setText(err_value)
+        finally:
+            progress.finish()
 
         if data is None:
             self.dataReport = None
         else:
             domain = data.domain
-            self.infoa.setText(
-                "{} instance(s), {} feature(s), {} meta attribute(s)"
-                .format(len(data), len(domain.attributes), len(domain.metas)))
+            text = "{} instance(s), {} feature(s), {} meta attribute(s)".format(
+                len(data), len(domain.attributes), len(domain.metas))
             if domain.has_continuous_class:
-                self.infob.setText("Regression; numerical class.")
+                text += "\nRegression; numerical class."
             elif domain.has_discrete_class:
-                self.infob.setText("Classification; " +
-                                   "discrete class with {} values."
-                                   .format(len(domain.class_var.values)))
+                text += "\nClassification; discrete class with {} values.".format(
+                    len(domain.class_var.values))
             elif data.domain.class_vars:
-                self.infob.setText("Multi-target; {} target variables."
-                                   .format(len(data.domain.class_vars)))
+                text += "\nMulti-target; {} target variables.".format(
+                    len(data.domain.class_vars))
             else:
-                self.infob.setText("Data has no target variable.")
+                text += "\nData has no target variable."
+            if 'Timestamp' in data.domain:
+                # Google Forms uses this header to timestamp responses
+                text += '\n\nFirst entry: {}\nLast entry: {}'.format(
+                    data[0, 'Timestamp'], data[-1, 'Timestamp'])
+            self.info.setText(text)
             self.warnings.setText("")
 
             add_origin(data, fn)
-            # make new data and send it
-            file_name = os.path.split(fn)[1]
-            if "." in file_name:
-                data.name = file_name[:file_name.rfind('.')]
-            else:
-                data.name = file_name
+
+            # Set title for URL paths
+            rp = self.recent_paths[0]
+            rp = self.recent_paths[0] = RecentPath(getattr(data, 'origin', rp.abspath),
+                                                   rp.prefix, rp.relpath, data.name)
+            # Ensure the same URL isn't in recent_paths twice
+            try: del self.recent_paths[self.recent_paths.index(rp, 1)]
+            except ValueError: pass
 
             self.dataReport = self.prepareDataReport(data)
         self.send("Data", data)


Test patch:
diff --git a/Orange/tests/test_table.py b/Orange/tests/test_table.py
--- a/Orange/tests/test_table.py
+++ b/Orange/tests/test_table.py
@@ -1261,6 +1261,27 @@ def test_load_from_url(self):
         np.testing.assert_array_equal(d1.X, d2.X)
         np.testing.assert_array_equal(d1.Y, d2.Y)
 
+    class _MockUrlOpen(MagicMock):
+        headers = {'content-disposition':
+            'attachment; filename="Something-FormResponses.tsv"; '
+            'filename*=UTF-8''Something%20%28Responses%29.tsv'}
+        url = 'https://docs.google.com/spreadsheets/d/ABCD/edit'
+        def __enter__(self): return self
+        def __exit__(self, *args, **kwargs): pass
+        def read(self): return b'''\
+a\tb\tc
+1\t2\t3
+2\t3\t4'''
+
+    urlopen = _MockUrlOpen()
+
+    @patch('Orange.data.table.urlopen', urlopen)
+    def test_google_sheets(self):
+        d = data.Table(self.urlopen.url)
+        self.urlopen.assert_called_with('https://docs.google.com/spreadsheets/d/ABCD/export?format=tsv', timeout=10)
+        self.assertEqual(len(d), 2)
+        self.assertEqual(d.name, 'Something-FormResponses')
+
 
 class CreateTableWithDomain(TableTests):
     def test_creates_an_empty_table_with_given_domain(self):