Repository: qutebrowser/qutebrowser

Issue:
Download fails to start on Windows if the filename contains forbidden characters
[List of all forbidden characters](https://msdn.microsoft.com/en-us/library/aa365247#naming_conventions)

qutebrowser doesn't give a warning and the download fails. Possible solution would be to remove them, or warn the user and ask for a new filename input?


Discussion:
How does it fail exactly, and with what filename? qutebrowser already has some code to replace invalid characters.
> 15:22:47 DEBUG    downloads  webenginedownloads:_on_state_changed:61 State for <qutebrowser.browser.webengine.webenginedownloads.DownloadItem basename='605748 kors k & REDALiCE - S2TBTANO*C Anthem feat. Yukacco.osz'> changed to DownloadInterrupted
15:22:47 DEBUG    message    message:_log_stack:46 Stack for error message:
  File "C:\Users\llyyr\AppData\Local\Programs\Python\Python36\Lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "C:\Users\llyyr\AppData\Local\Programs\Python\Python36\Lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\Program Files\qutebrowser\qutebrowser\__main__.py", line 29, in <module>
    sys.exit(qutebrowser.qutebrowser.main())
  File "C:\Program Files\qutebrowser\qutebrowser\qutebrowser.py", line 188, in main
    return app.run(args)
  File "C:\Program Files\qutebrowser\qutebrowser\app.py", line 139, in run
    ret = qt_mainloop()
  File "C:\Program Files\qutebrowser\qutebrowser\app.py", line 149, in qt_mainloop
    return qApp.exec_()
  File "C:\Program Files\qutebrowser\qutebrowser\browser\webengine\webenginedownloads.py", line 89, in _on_state_changed
    self._die(reason)
  File "C:\Program Files\qutebrowser\qutebrowser\browser\downloads.py", line 500, in _die
    self.error.emit(msg)
  File "C:\Program Files\qutebrowser\qutebrowser\browser\downloads.py", line 841, in _on_error
    message.error("Download error: {}".format(msg))
  File "C:\Program Files\qutebrowser\qutebrowser\utils\message.py", line 58, in error
    stack = traceback.format_stack()
15:22:47 ERROR    message    message:error:63 Download error: General file operation failure

The download works if I remove * from the filename.
Confirmed (on linux) that asterisks make it through to the saved file, checked on both backends. It doesn't look like sanitize_filename is being called from anywhere, there's just the encoding thing in `create_full_filename`. Not sure where it should be added in though, in `create_full_filename` probably.

Resolving patch:
diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -134,11 +134,12 @@ def create_full_filename(basename, filename):
     Return:
         The full absolute path, or None if filename creation was not possible.
     """
+    basename = utils.sanitize_filename(basename)
+    # Filename can be a full path so don't use sanitize_filename on it.
     # Remove chars which can't be encoded in the filename encoding.
     # See https://github.com/qutebrowser/qutebrowser/issues/427
     encoding = sys.getfilesystemencoding()
     filename = utils.force_encoding(filename, encoding)
-    basename = utils.force_encoding(basename, encoding)
     if os.path.isabs(filename) and (os.path.isdir(filename) or
                                     filename.endswith(os.sep)):
         # We got an absolute directory from the user, so we save it under
@@ -159,8 +160,7 @@ def get_filename_question(*, suggested_filename, url, parent=None):
         url: The URL the download originated from.
         parent: The parent of the question (a QObject).
     """
-    encoding = sys.getfilesystemencoding()
-    suggested_filename = utils.force_encoding(suggested_filename, encoding)
+    suggested_filename = utils.sanitize_filename(suggested_filename)
 
     q = usertypes.Question(parent)
     q.title = "Save file to:"
@@ -1224,8 +1224,7 @@ def get_tmpfile(self, suggested_name):
             A tempfile.NamedTemporaryFile that should be used to save the file.
         """
         tmpdir = self._get_tmpdir()
-        encoding = sys.getfilesystemencoding()
-        suggested_name = utils.force_encoding(suggested_name, encoding)
+        suggested_name = utils.sanitize_filename(suggested_name)
         # Make sure that the filename is not too long
         suggested_name = utils.elide_filename(suggested_name, 50)
         fobj = tempfile.NamedTemporaryFile(dir=tmpdir.name, delete=False,
diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py
--- a/qutebrowser/utils/utils.py
+++ b/qutebrowser/utils/utils.py
@@ -499,10 +499,22 @@ def sanitize_filename(name, replacement='_'):
     """
     if replacement is None:
         replacement = ''
-    # Bad characters taken from Windows, there are even fewer on Linux
+
+    # Remove chars which can't be encoded in the filename encoding.
+    # See https://github.com/qutebrowser/qutebrowser/issues/427
+    encoding = sys.getfilesystemencoding()
+    name = force_encoding(name, encoding)
+
     # See also
     # https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words
-    bad_chars = '\\/:*?"<>|'
+    if is_windows:
+        bad_chars = '\\/:*?"<>|'
+    elif is_mac:
+        # Colons can be confusing in finder https://superuser.com/a/326627
+        bad_chars = '/:'
+    else:
+        bad_chars = '/'
+
     for bad_char in bad_chars:
         name = name.replace(bad_char, replacement)
     return name


Test patch:
diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py
--- a/tests/helpers/fixtures.py
+++ b/tests/helpers/fixtures.py
@@ -459,28 +459,44 @@ def mode_manager(win_registry, config_stub, qapp):
     objreg.delete('mode-manager', scope='window', window=0)
 
 
-@pytest.fixture
-def config_tmpdir(monkeypatch, tmpdir):
+def standarddir_tmpdir(folder, monkeypatch, tmpdir):
     """Set tmpdir/config as the configdir.
 
     Use this to avoid creating a 'real' config dir (~/.config/qute_test).
     """
-    confdir = tmpdir / 'config'
+    confdir = tmpdir / folder
     confdir.ensure(dir=True)
-    monkeypatch.setattr(standarddir, 'config', lambda auto=False: str(confdir))
+    if hasattr(standarddir, folder):
+        monkeypatch.setattr(standarddir, folder,
+                            lambda **_kwargs: str(confdir))
     return confdir
 
 
+@pytest.fixture
+def download_tmpdir(monkeypatch, tmpdir):
+    """Set tmpdir/download as the downloaddir.
+
+    Use this to avoid creating a 'real' download dir (~/.config/qute_test).
+    """
+    return standarddir_tmpdir('download', monkeypatch, tmpdir)
+
+
+@pytest.fixture
+def config_tmpdir(monkeypatch, tmpdir):
+    """Set tmpdir/config as the configdir.
+
+    Use this to avoid creating a 'real' config dir (~/.config/qute_test).
+    """
+    return standarddir_tmpdir('config', monkeypatch, tmpdir)
+
+
 @pytest.fixture
 def data_tmpdir(monkeypatch, tmpdir):
     """Set tmpdir/data as the datadir.
 
     Use this to avoid creating a 'real' data dir (~/.local/share/qute_test).
     """
-    datadir = tmpdir / 'data'
-    datadir.ensure(dir=True)
-    monkeypatch.setattr(standarddir, 'data', lambda system=False: str(datadir))
-    return datadir
+    return standarddir_tmpdir('data', monkeypatch, tmpdir)
 
 
 @pytest.fixture
@@ -489,10 +505,7 @@ def runtime_tmpdir(monkeypatch, tmpdir):
 
     Use this to avoid creating a 'real' runtime dir.
     """
-    runtimedir = tmpdir / 'runtime'
-    runtimedir.ensure(dir=True)
-    monkeypatch.setattr(standarddir, 'runtime', lambda: str(runtimedir))
-    return runtimedir
+    return standarddir_tmpdir('runtime', monkeypatch, tmpdir)
 
 
 @pytest.fixture
@@ -501,10 +514,7 @@ def cache_tmpdir(monkeypatch, tmpdir):
 
     Use this to avoid creating a 'real' cache dir (~/.cache/qute_test).
     """
-    cachedir = tmpdir / 'cache'
-    cachedir.ensure(dir=True)
-    monkeypatch.setattr(standarddir, 'cache', lambda: str(cachedir))
-    return cachedir
+    return standarddir_tmpdir('cache', monkeypatch, tmpdir)
 
 
 @pytest.fixture
diff --git a/tests/unit/browser/webkit/test_downloads.py b/tests/unit/browser/webkit/test_downloads.py
--- a/tests/unit/browser/webkit/test_downloads.py
+++ b/tests/unit/browser/webkit/test_downloads.py
@@ -22,6 +22,14 @@
 from qutebrowser.browser import downloads, qtnetworkdownloads
 
 
+@pytest.fixture(autouse=True)
+def is_platform(monkeypatch, platform="windows"):
+    for p in ["mac", "linux", "posix", "windows"]:
+        monkeypatch.setattr(
+            'qutebrowser.utils.utils.is_{}'.format(p),
+            p == platform)
+
+
 def test_download_model(qapp, qtmodeltester, config_stub, cookiejar_and_cache,
                         fake_args):
     """Simple check for download model internals."""
@@ -96,3 +104,27 @@ def test_openfile_custom_command(self):
     ])
     def test_class_hierarchy(self, obj):
         assert isinstance(obj, downloads._DownloadTarget)
+
+
+@pytest.mark.parametrize('raw, platform, expected', [
+    ('http://foo/bar', 'windows', 'bar'),
+    ('A *|<>\\: bear!', 'windows', 'A ______ bear!'),
+    ('A *|<>\\: bear!', 'posix', 'A *|<>\\: bear!')
+])
+def test_sanitized_filenames(raw, platform, expected, config_stub,
+                             download_tmpdir, monkeypatch):
+    is_platform(monkeypatch, platform)
+    manager = downloads.AbstractDownloadManager()
+    target = downloads.FileDownloadTarget(str(download_tmpdir))
+    item = downloads.AbstractDownloadItem()
+
+    # Don't try to start a timer outside of a QThread
+    manager._update_timer.isActive = lambda: True
+
+    # Abstract methods
+    item._ensure_can_set_filename = lambda *args: True
+    item._after_set_filename = lambda *args: True
+
+    manager._init_item(item, True, raw)
+    item.set_target(target)
+    assert item._filename.endswith(expected)
diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py
--- a/tests/unit/utils/test_utils.py
+++ b/tests/unit/utils/test_utils.py
@@ -113,6 +113,14 @@ def test_elided(self, filename, length, expected):
         assert utils.elide_filename(filename, length) == expected
 
 
+@pytest.fixture(autouse=True)
+def is_platform(monkeypatch, platform="windows"):
+    for p in ["mac", "linux", "posix", "windows"]:
+        monkeypatch.setattr(
+            'qutebrowser.utils.utils.is_{}'.format(p),
+            p == platform)
+
+
 @pytest.fixture(params=[True, False])
 def freezer(request, monkeypatch):
     if request.param and not getattr(sys, 'frozen', False):
@@ -629,12 +637,15 @@ def test_force_encoding(inp, enc, expected):
     assert utils.force_encoding(inp, enc) == expected
 
 
-@pytest.mark.parametrize('inp, expected', [
-    ('normal.txt', 'normal.txt'),
-    ('user/repo issues.mht', 'user_repo issues.mht'),
-    ('<Test\\File> - "*?:|', '_Test_File_ - _____'),
+@pytest.mark.parametrize('inp, platform, expected', [
+    ('normal.txt', 'windows', 'normal.txt'),
+    ('user/repo issues.mht', 'windows', 'user_repo issues.mht'),
+    ('<Test\\File> - "*?:|', 'windows', '_Test_File_ - _____'),
+    ('<Test\\File> - "*?:|', 'mac', '<Test\\File> - "*?_|'),
+    ('<Test\\File> - "*?:|', 'posix', '<Test\\File> - "*?:|'),
 ])
-def test_sanitize_filename(inp, expected):
+def test_sanitize_filename(inp, platform, expected, monkeypatch):
+    is_platform(monkeypatch, platform)
     assert utils.sanitize_filename(inp) == expected