Repository: sunpy/sunpy

Issue:
{file} template can produce illegal file names on windows.
When using sunpy 0.1 (4910875ee92884b608b05caeff96f45de729a1b4) on a windows 7 system, the {file} template under some conditions produces illegal file names. This causes the download to fail with the error message "invalid mode ('wb') or filename". This bug will not occur on other operating systems that allow these characters in filenames.

On windows systems the following characters are illegal: <>:"/|?*^ (and the null character). The function sunpy.net.vso.VSOClient.mk_filename produces filenames containing these characters. An example is aia 192Å from jsoc, which contains colon (:) in the time format. Debugging mk_filename shows that the filenames are of the format
aia.lev1.193A_2011-09-20T01:00:07.84Z.image_lev1.fits

Downloading from another source, e.g. eit, with another filename convention succeeds with my setup.
## Code to replicate:

```
import os
from sunpy.net import vso
client = vso.VSOClient()
queryresult = client.query(
    vso.attrs.Time((2011, 9, 20, 1), (2011, 9, 20, 1, 1)),
    vso.attrs.Instrument('aia'),
    vso.attrs.Provider('jsoc'),
    vso.attrs.Wave(192,196)
)
print("Number of records found: %d " % queryresult.num_records())
# Prints 5
getter = client.get(queryresult, path="output/{file}")
res = getter.wait()

#>>> res
#[]

#>>> getter.errors
#[IOError(22, "invalid mode ('wb') or filename"),
# IOError(22, "invalid mode ('wb') or filename"),
# IOError(22, "invalid mode ('wb') or filename"),
# IOError(22, "invalid mode ('wb') or filename"),
# IOError(22, "invalid mode ('wb') or filename")]
```
## Workaround:

Do not include {file} in the path template, which causes files to be named by incremental numbers instead. You must supply a file template indicating the directory to download into, because the default templete includes {file}.
## Proposed fix:

A filename cleaning function that converts illegal characters e.g. into underscores. This can either be done on all operating systems or just on windows. Doing it on all operating systems keeps naming consistent between platforms, but causes naming to change from old expected behaviour, which might break existing code. Doing it only on windows is a compromise that avoids this problem.



Discussion:
http://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename-in-python

The comments there have some interesting caveats that might have to be considered when fixing this.


Resolving patch:
diff --git a/sunpy/net/download.py b/sunpy/net/download.py
--- a/sunpy/net/download.py
+++ b/sunpy/net/download.py
@@ -36,6 +36,7 @@ def __init__(self, max_conn=5, max_total=20):
         self.buf = 9096
 
         self.done_lock = threading.Semaphore(0)
+        self.mutex = threading.Lock()
     
     def _start_download(self, url, path, callback, errback):
         try:
@@ -51,17 +52,20 @@ def _start_download(self, url, path, callback, errback):
                     while True:
                         rec = sock.read(self.buf)
                         if not rec:
-                            self._close(callback, [{'path': fullname}], server)
+                            with self.mutex:
+                                self._close(callback, [{'path': fullname}], server)
                             break
                         else:
                             fd.write(rec)
         except Exception, e:
             if errback is not None:
-                errback(e)
-    
+                with self.mutex:
+                    self._close(errback, [e], server)
+
     def _attempt_download(self, url, path, callback, errback):
         """ Attempt download. If max. connection limit reached, queue for download later.
         """
+
         num_connections = self.connections[self._get_server(url)]
         
         # If max downloads has not been exceeded, begin downloading
@@ -147,13 +151,13 @@ def _close(self, callback, args, server):
         """ Called after download is done. Activated queued downloads, call callback.
         """
         callback(*args)
-        
+
+        self.connections[server] -= 1
+        self.conns -= 1
+
         if self.q[server]:
-            self._start_download(*self.q[server].pop())
+            self._attempt_download(*self.q[server].pop())
         else:
-            self.connections[server] -= 1
-            self.conns -= 1
-            
             for k, v in self.q.iteritems():  # pylint: disable=W0612
                 while v:
                     if self._attempt_download(*v[0]):
@@ -162,33 +166,3 @@ def _close(self, callback, args, server):
                             return
                     else:
                         break
-
-
-if __name__ == '__main__':
-    import tempfile
-
-    def wait_for(n, callback):  # pylint: disable=W0613
-        items = []
-
-        def _fun(handler):
-            print handler
-            items.append(handler)
-            if len(items) == n:
-                callback(items)
-        return _fun
-
-    tmp = tempfile.mkdtemp()
-    print tmp
-    path_fun = partial(default_name, tmp)
-    
-    dw = Downloader(1, 2)
-    
-    on_finish = wait_for(4, lambda _: dw.stop())
-    dw.download('ftp://speedtest.inode.at/speedtest-5mb', path_fun, on_finish)
-    dw.download('ftp://speedtest.inode.at/speedtest-20mb', path_fun, on_finish)
-    dw.download('https://bitsrc.org', path_fun, on_finish)
-    dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish)
-    
-    print dw.conns
-    
-    dw.wait()
diff --git a/sunpy/net/vso/vso.py b/sunpy/net/vso/vso.py
--- a/sunpy/net/vso/vso.py
+++ b/sunpy/net/vso/vso.py
@@ -29,8 +29,10 @@
 from sunpy.net import download
 from sunpy.util.net import get_filename, slugify
 from sunpy.net.attr import and_, Attr
+from sunpy.net.vso import attrs
 from sunpy.net.vso.attrs import walker, TIMEFORMAT
 from sunpy.util import to_angstrom, print_table, replacement_filename
+from sunpy.util.net import slugify
 from sunpy.time import parse_time
 
 DEFAULT_URL = 'http://docs.virtualsolar.org/WSDL/VSOi_rpc_literal.wsdl'
@@ -345,7 +347,7 @@ def mk_filename(pattern, response, sock, url, overwrite=False):
         fs_encoding = sys.getfilesystemencoding()
         if fs_encoding is None:
             fs_encoding = "ascii"
-        name = name.encode(fs_encoding, "ignore")
+        name = slugify(name).encode(fs_encoding, "ignore")
 
         if not name:
             name = "file"
@@ -355,7 +357,7 @@ def mk_filename(pattern, response, sock, url, overwrite=False):
         if not overwrite and os.path.exists(fname):
             fname = replacement_filename(fname)
         
-        dir_ = os.path.dirname(fname)
+        dir_ = os.path.abspath(os.path.dirname(fname))
         if not os.path.exists(dir_):
             os.makedirs(dir_)
         return fname
diff --git a/sunpy/util/net.py b/sunpy/util/net.py
--- a/sunpy/util/net.py
+++ b/sunpy/util/net.py
@@ -21,7 +21,7 @@
 from sunpy.util import replacement_filename
 
 # Characters not allowed in slugified version.
-_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')
+_punct_re = re.compile(r'[:\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')
 
 def slugify(text, delim=u'_', encoding="ascii"):
     """ Slugify given unicode text. """
@@ -74,6 +74,15 @@ def get_system_filename(sock, url, default=u"file"):
     return name.encode(sys.getfilesystemencoding(), 'ignore')
 
 
+def get_system_filename_slugify(sock, url, default=u"file"):
+    """ Get filename from given urllib2.urlopen object and URL.
+    First, attempts to extract Content-Disposition, second, extract
+    from URL, eventually fall back to default. Returns bytestring
+    in file system encoding, normalized so it shouldn't violate
+    operating system restrictions. """
+    return slugify(get_system_filename(sock, url, default))
+
+
 def download_file(url, directory, default=u'file', overwrite=False):
     """ Download file from url into directory. Try to get filename from
     Content-Disposition header, otherwise get from path of url. Fall


Test patch:
diff --git a/sunpy/tests/net/test_download.py b/sunpy/tests/net/test_download.py
--- a/sunpy/tests/net/test_download.py
+++ b/sunpy/tests/net/test_download.py
@@ -9,11 +9,35 @@
 
 import os
 import tempfile
+import threading
 
 from functools import partial
 
+import sunpy
+
 from sunpy.net.download import Downloader, default_name
-import threading
+
+
+class CalledProxy(object):
+    def __init__(self, fn):
+        self.fn = fn
+        self.fired = False
+
+    def __call__(self, *args, **kwargs):
+        self.fn(*args, **kwargs)
+        self.fired = True
+
+
+class MockConfig(object):
+    def __init__(self):
+        self.dct = {}
+
+    def add_section(self, name, dct):
+        self.dct[name] = dct
+
+    def get(self, one, other):
+        return self.dct[one][other]
+
 
 def wait_for(n, callback): #pylint: disable=W0613
     items = []
@@ -44,25 +68,88 @@ def test_path_exception():
 
 def test_download_http():
     items = []
+    lck = threading.Lock()
 
     def wait_for(n, callback):  # pylint: disable=W0613
         def _fun(handler):
-            items.append(handler)
-            if len(items) == n:
-                callback(items)
+            with lck:
+                items.append(handler)
+                if len(items) == n:
+                    callback(items)
         return _fun
 
     tmp = tempfile.mkdtemp()
     path_fun = partial(default_name, tmp)
 
     dw = Downloader(1, 1)
+    _stop = lambda _: dw.stop()
+
+    timeout = CalledProxy(_stop)
+    threading.Timer(60, timeout).start()
 
-    on_finish = wait_for(2, lambda _: dw.stop())
+    on_finish = wait_for(3, lambda _: dw.stop())
     dw.download('http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', path_fun, on_finish)
     dw.download('http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js', path_fun, on_finish)
+    dw.download('https://raw.github.com/sunpy/sunpy/master/INSTALL.txt', path_fun, on_finish)
     # dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish)
 
     dw.wait()
 
+    assert len(items) == 3
+    assert not timeout.fired
+
     for item in items:
-        assert os.path.exists(item ['path'])
+        assert os.path.exists(item['path'])
+
+
+def test_download_default_dir():
+    _config = sunpy.config
+
+    try:
+        tmpdir = tempfile.mkdtemp()
+
+        sunpy.config = MockConfig()
+        sunpy.config.add_section(
+            "downloads", {"download_dir": tmpdir}
+        )
+
+        dw = Downloader(1, 1)
+        _stop = lambda _: dw.stop()
+
+        timeout = CalledProxy(_stop)
+        errback = CalledProxy(_stop)
+        dw.download(
+            'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
+            callback=_stop,
+            errback=errback
+        )
+
+        threading.Timer(10, timeout).start()
+        dw.wait()
+
+        assert not timeout.fired
+        assert not errback.fired
+        assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
+    finally:
+        sunpy.config = _config
+
+def test_download_dir():
+    tmpdir = tempfile.mkdtemp()
+
+    dw = Downloader(1, 1)
+    _stop = lambda _: dw.stop()
+    timeout = CalledProxy(_stop)
+    errback = CalledProxy(_stop)
+
+    dw.download(
+        'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
+        tmpdir,
+        callback=_stop,
+        errback=errback
+    )
+
+    threading.Timer(10, timeout).start()
+    dw.wait()
+    assert not timeout.fired
+    assert not errback.fired
+    assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))