Repository: mne-tools/mne-python

Issue:
BUG: partially downloaded dataset
I was trying to update the testing dataset but I already had a partially downloaded zip file in my mne/examples/ directory. This made the downloading hang. I tried

``` py
from mne.datasets import testing
testing.data_path(force_update=True)
```

I am not sure if this happens with other datasets. And also I'm not sure if there's an easy way to replicate this because you would need to have a partially downloaded copy of the old dataset (before it was updated). 



Discussion:
Usually partial downloads work, but for `testing` they might not since the hashes are not the same download-to-download. So one solution is to add a parameter to our dataset downloaders that says whether it can be resumed, which would be True for all datasets except the testing one.

If it cannot be resumed, then why not simply download the entire dataset again instead of giving an incorrect logging message? Do we even need a new parameter?
![bug_downloading](https://cloud.githubusercontent.com/assets/15852194/11865492/9b28feda-a4a4-11e5-8123-6574ca7bfc55.png)
My screen has been stuck here for more than 2 minutes. First, it tells me (incorrectly) that there is only 1 byte to download and second, it is stuck at 100% ...

Most datasets _can_ be resumed. So in our private functions somewhere we
need to indicate that all but the testing set can be resumed, and modify
our code appropriately. I assume it would be done by adding a parameter
somewhere. That's what I meant by adding a parameter.

Github also does not report the file size correctly, so that's why it
appears to sit there. PR welcome if you can find a fix.

okay, let's keep the issue open. I'll get around to it when I have a better internet connection (which is also the reason I'm complaining a lot -- don't have the luxury to wait for another failed download). For now, I'll try to get the testing data using the browser ...

One thing we could do that would probably make it resumable is make "releases" of the testing dataset. This will require simultaneous updates of `mne-python` (to say which version we need) and `mne-testing-updates`, but I think that's okay.

I can give it a shot now actually, if what I'm thinking works it should be pretty quick. And my internet connection is stable enough that it shouldn't be too painful.


Resolving patch:
diff --git a/mne/datasets/utils.py b/mne/datasets/utils.py
--- a/mne/datasets/utils.py
+++ b/mne/datasets/utils.py
@@ -160,11 +160,18 @@ def _data_path(path=None, force_update=False, update_path=True, download=True,
            }[name]
 
     path = _get_path(path, key, name)
+    # To update the testing dataset, push commits, then make a new release
+    # on GitHub. Then update the "testing_release" variable:
+    testing_release = '0.11'
+    # And also update the "hashes['testing']" variable below.
+
+    # To update any other dataset, update the data archive itself (upload
+    # an updated version) and update the hash.
     archive_names = dict(
         sample='MNE-sample-data-processed.tar.gz',
         spm='MNE-spm-face.tar.bz2',
         somato='MNE-somato-data.tar.gz',
-        testing='mne-testing-data-master.tar.gz',
+        testing='mne-testing-data-%s.tar.gz' % testing_release,
         fake='foo.tgz',
     )
     if archive_name is not None:
@@ -182,8 +189,8 @@ def _data_path(path=None, force_update=False, update_path=True, download=True,
         spm='https://s3.amazonaws.com/mne-python/datasets/%s',
         somato='https://s3.amazonaws.com/mne-python/datasets/%s',
         brainstorm='https://copy.com/ZTHXXFcuIZycvRoA/brainstorm/%s',
-        testing='https://github.com/mne-tools/mne-testing-data/archive/'
-                'master.tar.gz',
+        testing='https://codeload.github.com/mne-tools/mne-testing-data/'
+                'tar.gz/%s' % testing_release,
         fake='https://github.com/mne-tools/mne-testing-data/raw/master/'
              'datasets/%s',
     )
@@ -192,11 +199,11 @@ def _data_path(path=None, force_update=False, update_path=True, download=True,
         spm='3e9e83c642136e5b720e2ecc5dcc3244',
         somato='f3e3a8441477bb5bacae1d0c6e0964fb',
         brainstorm=None,
-        testing=None,
+        testing='d1753ce154e0e6af12f1b82b21e975ce',
         fake='3194e9f7b46039bb050a74f3e1ae9908',
     )
     folder_origs = dict(  # not listed means None
-        testing='mne-testing-data-master',
+        testing='mne-testing-data-%s' % testing_release,
     )
     folder_name = folder_names[name]
     archive_name = archive_names[name]
diff --git a/mne/utils.py b/mne/utils.py
--- a/mne/utils.py
+++ b/mne/utils.py
@@ -1233,7 +1233,7 @@ class ProgressBar(object):
     def __init__(self, max_value, initial_value=0, mesg='', max_chars=40,
                  progress_character='.', spinner=False, verbose_bool=True):
         self.cur_value = initial_value
-        self.max_value = float(max_value)
+        self.max_value = max_value
         self.mesg = mesg
         self.max_chars = max_chars
         self.progress_character = progress_character
@@ -1265,6 +1265,9 @@ def update(self, cur_value, mesg=None):
 
         # Update the message
         if mesg is not None:
+            if mesg == 'file_sizes':
+                mesg = '(%s / %s)' % (sizeof_fmt(self.cur_value),
+                                      sizeof_fmt(self.max_value))
             self.mesg = mesg
 
         # The \r tells the cursor to return to the beginning of the line rather
@@ -1303,55 +1306,8 @@ def update_with_increment_value(self, increment_value, mesg=None):
         self.update(self.cur_value, mesg)
 
 
-def _chunk_read(response, local_file, initial_size=0, verbose_bool=True):
-    """Download a file chunk by chunk and show advancement
-
-    Can also be used when resuming downloads over http.
-
-    Parameters
-    ----------
-    response: urllib.response.addinfourl
-        Response to the download request in order to get file size.
-    local_file: file
-        Hard disk file where data should be written.
-    initial_size: int, optional
-        If resuming, indicate the initial size of the file.
-
-    Notes
-    -----
-    The chunk size will be automatically adapted based on the connection
-    speed.
-    """
-    # Adapted from NISL:
-    # https://github.com/nisl/tutorial/blob/master/nisl/datasets.py
-
-    # Returns only amount left to download when resuming, not the size of the
-    # entire file
-    total_size = int(response.headers.get('Content-Length', '1').strip())
-    total_size += initial_size
-
-    progress = ProgressBar(total_size, initial_value=initial_size,
-                           max_chars=40, spinner=True, mesg='downloading',
-                           verbose_bool=verbose_bool)
-    chunk_size = 8192  # 2 ** 13
-    while True:
-        t0 = time.time()
-        chunk = response.read(chunk_size)
-        dt = time.time() - t0
-        if dt < 0.001:
-            chunk_size *= 2
-        elif dt > 0.5 and chunk_size > 8192:
-            chunk_size = chunk_size // 2
-        if not chunk:
-            if verbose_bool:
-                sys.stdout.write('\n')
-                sys.stdout.flush()
-            break
-        _chunk_write(chunk, local_file, progress)
-
-
-def _chunk_read_ftp_resume(url, temp_file_name, local_file, verbose_bool=True):
-    """Resume downloading of a file from an FTP server"""
+def _get_ftp(url, temp_file_name, initial_size, file_size, verbose_bool):
+    """Safely (resume a) download to a file from FTP"""
     # Adapted from: https://pypi.python.org/pypi/fileDownloader.py
     # but with changes
 
@@ -1359,7 +1315,6 @@ def _chunk_read_ftp_resume(url, temp_file_name, local_file, verbose_bool=True):
     file_name = os.path.basename(parsed_url.path)
     server_path = parsed_url.path.replace(file_name, "")
     unquoted_server_path = urllib.parse.unquote(server_path)
-    local_file_size = os.path.getsize(temp_file_name)
 
     data = ftplib.FTP()
     if parsed_url.port is not None:
@@ -1370,23 +1325,75 @@ def _chunk_read_ftp_resume(url, temp_file_name, local_file, verbose_bool=True):
     if len(server_path) > 1:
         data.cwd(unquoted_server_path)
     data.sendcmd("TYPE I")
-    data.sendcmd("REST " + str(local_file_size))
+    data.sendcmd("REST " + str(initial_size))
     down_cmd = "RETR " + file_name
-    file_size = data.size(file_name)
-    progress = ProgressBar(file_size, initial_value=local_file_size,
-                           max_chars=40, spinner=True, mesg='downloading',
+    assert file_size == data.size(file_name)
+    progress = ProgressBar(file_size, initial_value=initial_size,
+                           max_chars=40, spinner=True, mesg='file_sizes',
                            verbose_bool=verbose_bool)
 
     # Callback lambda function that will be passed the downloaded data
     # chunk and will write it to file and update the progress bar
-    def chunk_write(chunk):
-        return _chunk_write(chunk, local_file, progress)
-    data.retrbinary(down_cmd, chunk_write)
-    data.close()
+    mode = 'ab' if initial_size > 0 else 'wb'
+    with open(temp_file_name, mode) as local_file:
+        def chunk_write(chunk):
+            return _chunk_write(chunk, local_file, progress)
+        data.retrbinary(down_cmd, chunk_write)
+        data.close()
     sys.stdout.write('\n')
     sys.stdout.flush()
 
 
+def _get_http(url, temp_file_name, initial_size, file_size, verbose_bool):
+    """Safely (resume a) download to a file from http(s)"""
+    # Actually do the reading
+    req = urllib.request.Request(url)
+    if initial_size > 0:
+        req.headers['Range'] = 'bytes=%s-' % (initial_size,)
+    try:
+        response = urllib.request.urlopen(req)
+    except Exception:
+        # There is a problem that may be due to resuming, some
+        # servers may not support the "Range" header. Switch
+        # back to complete download method
+        logger.info('Resuming download failed (server '
+                    'rejected the request). Attempting to '
+                    'restart downloading the entire file.')
+        del req.headers['Range']
+        response = urllib.request.urlopen(req)
+    total_size = int(response.headers.get('Content-Length', '1').strip())
+    if initial_size > 0 and file_size == total_size:
+        logger.info('Resuming download failed (resume file size '
+                    'mismatch). Attempting to restart downloading the '
+                    'entire file.')
+        initial_size = 0
+    total_size += initial_size
+    if total_size != file_size:
+        raise RuntimeError('URL could not be parsed properly')
+    mode = 'ab' if initial_size > 0 else 'wb'
+    progress = ProgressBar(total_size, initial_value=initial_size,
+                           max_chars=40, spinner=True, mesg='file_sizes',
+                           verbose_bool=verbose_bool)
+    chunk_size = 8192  # 2 ** 13
+    with open(temp_file_name, mode) as local_file:
+        while True:
+            t0 = time.time()
+            chunk = response.read(chunk_size)
+            dt = time.time() - t0
+            if dt < 0.005:
+                chunk_size *= 2
+            elif dt > 0.1 and chunk_size > 8192:
+                chunk_size = chunk_size // 2
+            if not chunk:
+                if verbose_bool:
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+                break
+            local_file.write(chunk)
+            progress.update_with_increment_value(len(chunk),
+                                                 mesg='file_sizes')
+
+
 def _chunk_write(chunk, local_file, progress):
     """Write a chunk to file and update the progress bar"""
     local_file.write(chunk)
@@ -1424,11 +1431,13 @@ def _fetch_file(url, file_name, print_destination=True, resume=True,
         raise ValueError('Bad hash value given, should be a 32-character '
                          'string:\n%s' % (hash_,))
     temp_file_name = file_name + ".part"
-    local_file = None
-    initial_size = 0
     verbose_bool = (logger.level <= 20)  # 20 is info
     try:
-        # Checking file size and displaying it alongside the download url
+        # Check file size and displaying it alongside the download url
+        u = urllib.request.urlopen(url, timeout=timeout)
+        u.close()
+        # this is necessary to follow any redirects
+        url = u.geturl()
         u = urllib.request.urlopen(url, timeout=timeout)
         try:
             file_size = int(u.headers.get('Content-Length', '1').strip())
@@ -1437,46 +1446,28 @@ def _fetch_file(url, file_name, print_destination=True, resume=True,
             del u
         logger.info('Downloading data from %s (%s)\n'
                     % (url, sizeof_fmt(file_size)))
-        # Downloading data
-        if resume and os.path.exists(temp_file_name):
-            local_file = open(temp_file_name, "ab")
-            # Resuming HTTP and FTP downloads requires different procedures
-            scheme = urllib.parse.urlparse(url).scheme
-            if scheme in ('http', 'https'):
-                local_file_size = os.path.getsize(temp_file_name)
-                # If the file exists, then only download the remainder
-                req = urllib.request.Request(url)
-                req.headers["Range"] = "bytes=%s-" % local_file_size
-                try:
-                    data = urllib.request.urlopen(req)
-                except Exception:
-                    # There is a problem that may be due to resuming, some
-                    # servers may not support the "Range" header. Switch back
-                    # to complete download method
-                    logger.info('Resuming download failed. Attempting to '
-                                'restart downloading the entire file.')
-                    local_file.close()
-                    _fetch_file(url, file_name, resume=False)
-                else:
-                    _chunk_read(data, local_file, initial_size=local_file_size,
-                                verbose_bool=verbose_bool)
-                    data.close()
-                    del data  # should auto-close
-            else:
-                _chunk_read_ftp_resume(url, temp_file_name, local_file,
-                                       verbose_bool=verbose_bool)
+
+        # Triage resume
+        if not os.path.exists(temp_file_name):
+            resume = False
+        if resume:
+            with open(temp_file_name, 'rb', buffering=0) as local_file:
+                local_file.seek(0, 2)
+                initial_size = local_file.tell()
+            del local_file
         else:
-            local_file = open(temp_file_name, "wb")
-            data = urllib.request.urlopen(url)
-            try:
-                _chunk_read(data, local_file, initial_size=initial_size,
-                            verbose_bool=verbose_bool)
-            finally:
-                data.close()
-                del data  # should auto-close
-        # temp file must be closed prior to the move
-        if not local_file.closed:
-            local_file.close()
+            initial_size = 0
+        # This should never happen if our functions work properly
+        if initial_size >= file_size:
+            raise RuntimeError('Local file (%s) is larger than remote '
+                               'file (%s), cannot resume download'
+                               % (sizeof_fmt(initial_size),
+                                  sizeof_fmt(file_size)))
+
+        scheme = urllib.parse.urlparse(url).scheme
+        fun = _get_http if scheme in ('http', 'https') else _get_ftp
+        fun(url, temp_file_name, initial_size, file_size, verbose_bool)
+
         # check md5sum
         if hash_ is not None:
             logger.info('Verifying download hash.')
@@ -1488,15 +1479,10 @@ def _fetch_file(url, file_name, print_destination=True, resume=True,
         shutil.move(temp_file_name, file_name)
         if print_destination is True:
             logger.info('File saved as %s.\n' % file_name)
-    except Exception as e:
+    except Exception:
         logger.error('Error while fetching file %s.'
                      ' Dataset fetching aborted.' % url)
-        logger.error("Error: %s", e)
         raise
-    finally:
-        if local_file is not None:
-            if not local_file.closed:
-                local_file.close()
 
 
 def sizeof_fmt(num):


Test patch:
diff --git a/mne/tests/test_utils.py b/mne/tests/test_utils.py
--- a/mne/tests/test_utils.py
+++ b/mne/tests/test_utils.py
@@ -382,7 +382,8 @@ def test_fetch_file():
     with ArgvSetter(disable_stderr=False):  # to capture stdout
         for url in urls:
             archive_name = op.join(tempdir, "download_test")
-            _fetch_file(url, archive_name, timeout=30., verbose=False)
+            _fetch_file(url, archive_name, timeout=30., verbose=False,
+                        resume=False)
             assert_raises(Exception, _fetch_file, 'NOT_AN_ADDRESS',
                           op.join(tempdir, 'test'), verbose=False)
             resume_name = op.join(tempdir, "download_resume")