Repository: pallets/werkzeug

Issue:
IDNA encoding for netloc in url_fix ?
i was considering using this for generating canonical urls, and ran across a limitation for my needs.

this might overstep the goals of this function, but I think it's in keeping with the "spirit" , as this is how most browsers actually behave.

given a unicode domain name with non-ascii chars, `url_fix` with encode the netloc with a percent encoding.  this is perfectly valid; however most web browsers have opted for the IDNA encoding and act as follows: if a percent-encoded domain is entered, the url is automatically rewritten as an IDNA encoded representation and then requested.



Discussion:
Seems okay to me.


Resolving patch:
diff --git a/werkzeug/urls.py b/werkzeug/urls.py
--- a/werkzeug/urls.py
+++ b/werkzeug/urls.py
@@ -8,6 +8,7 @@
     :copyright: (c) 2014 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
+import os
 import re
 from werkzeug._compat import text_type, PY2, to_unicode, \
      to_native, implements_to_string, try_coerce_native, \
@@ -174,6 +175,64 @@ def to_iri_tuple(self):
         """
         return url_parse(uri_to_iri(self))
 
+    def get_file_location(self, pathformat=None):
+        """Returns a tuple with the location of the file in the form
+        ``(server, location)``.  If the netloc is empty in the URL or
+        points to localhost, it's represented as ``None``.
+
+        The `pathformat` by default is autodetection but needs to be set
+        when working with URLs of a specific system.  The supported values
+        are ``'windows'`` when working with Windows or DOS paths and
+        ``'posix'`` when working with posix paths.
+
+        If the URL does not point to to a local file, the server and location
+        are both represented as ``None``.
+
+        :param pathformat: The expected format of the path component.
+                           Currently ``'windows'`` and ``'posix'`` are
+                           supported.  Defaults to ``None`` which is
+                           autodetect.
+        """
+        if self.scheme != 'file':
+            return None, None
+
+        path = url_unquote(self.path)
+        host = self.netloc or None
+
+        if pathformat is None:
+            if os.name == 'nt':
+                pathformat = 'windows'
+            else:
+                pathformat = 'posix'
+
+        if pathformat == 'windows':
+            if path[:1] == '/' and path[1:2].isalpha() and path[2:3] in '|:':
+                path = path[1:2] + ':' + path[3:]
+            windows_share = path[:3] in ('\\' * 3, '/' * 3)
+            import ntpath
+            path = ntpath.normpath(path)
+            # Windows shared drives are represented as ``\\host\\directory``.
+            # That results in a URL like ``file://///host/directory``, and a
+            # path like ``///host/directory``. We need to special-case this
+            # because the path contains the hostname.
+            if windows_share and host is None:
+                parts = path.lstrip('\\').split('\\', 1)
+                if len(parts) == 2:
+                    host, path = parts
+                else:
+                    host = parts[0]
+                    path = ''
+        elif pathformat == 'posix':
+            import posixpath
+            path = posixpath.normpath(path)
+        else:
+            raise TypeError('Invalid path format %s' % repr(pathformat))
+
+        if host in ('127.0.0.1', '::1', 'localhost'):
+            host = None
+
+        return host, path
+
     def _split_netloc(self):
         if self._at in self.netloc:
             return self.netloc.split(self._at, 1)
@@ -237,7 +296,7 @@ def encode_netloc(self):
         ]))
         if auth:
             rv = '%s@%s' % (auth, rv)
-        return rv.encode('ascii')
+        return to_native(rv)
 
     def encode(self, charset='utf-8', errors='replace'):
         """Encodes the URL to a tuple made out of bytes.  The charset is
@@ -491,10 +550,22 @@ def url_fix(s, charset='utf-8'):
     :param charset: The target charset for the URL if the url was given as
                     unicode string.
     """
-    scheme, netloc, path, qs, anchor = url_parse(to_unicode(s, charset, 'replace'))
-    path = url_quote(path, charset, safe='/%+$!*\'(),')
-    qs = url_quote_plus(qs, charset, safe=':&%=+$!*\'(),')
-    return to_native(url_unparse((scheme, netloc, path, qs, anchor)))
+    # First step is to switch to unicode processing and to convert
+    # backslashes (which are invalid in URLs anyways) to slashes.  This is
+    # consistent with what Chrome does.
+    s = to_unicode(s, charset, 'replace').replace('\\', '/')
+
+    # For the specific case that we look like a malformed windows URL
+    # we want to fix this up manually:
+    if s.startswith('file://') and s[7:8].isalpha() and s[8:10] in (':/', '|/'):
+        s = 'file:///' + s[7:]
+
+    url = url_parse(s)
+    path = url_quote(url.path, charset, safe='/%+$!*\'(),')
+    qs = url_quote_plus(url.query, charset, safe=':&%=+$!*\'(),')
+    anchor = url_quote_plus(url.fragment, charset, safe=':&%=+$!*\'(),')
+    return to_native(url_unparse((url.scheme, url.encode_netloc(),
+                                  path, qs, anchor)))
 
 
 def uri_to_iri(uri, charset='utf-8', errors='replace'):
@@ -585,7 +656,7 @@ def iri_to_uri(iri, charset='utf-8', errors='strict', safe_conversion=False):
 
     iri = url_parse(to_unicode(iri, charset, errors))
 
-    netloc = iri.encode_netloc().decode('ascii')
+    netloc = iri.encode_netloc()
     path = url_quote(iri.path, charset, errors, '/:~+%')
     query = url_quote(iri.query, charset, errors, '%&[]:;$*()+,!?*/=')
     fragment = url_quote(iri.fragment, charset, errors, '=%&[]:;$()+,!?*/')


Test patch:
diff --git a/tests/test_urls.py b/tests/test_urls.py
--- a/tests/test_urls.py
+++ b/tests/test_urls.py
@@ -17,6 +17,46 @@
 from werkzeug._compat import text_type, NativeStringIO, BytesIO
 
 
+def test_parsing():
+    url = urls.url_parse('http://anon:hunter2@[2001:db8:0:1]:80/a/b/c')
+    assert url.netloc == 'anon:hunter2@[2001:db8:0:1]:80'
+    assert url.username == 'anon'
+    assert url.password == 'hunter2'
+    assert url.port == 80
+    assert url.ascii_host == '2001:db8:0:1'
+
+    assert url.get_file_location() == (None, None)  # no file scheme
+
+
+@pytest.mark.parametrize('implicit_format', (True, False))
+@pytest.mark.parametrize('localhost', ('127.0.0.1', '::1', 'localhost'))
+def test_fileurl_parsing_windows(implicit_format, localhost, monkeypatch):
+    if implicit_format:
+        pathformat = None
+        monkeypatch.setattr('os.name', 'nt')
+    else:
+        pathformat = 'windows'
+        monkeypatch.delattr('os.name')  # just to make sure it won't get used
+
+    url = urls.url_parse('file:///C:/Documents and Settings/Foobar/stuff.txt')
+    assert url.netloc == ''
+    assert url.scheme == 'file'
+    assert url.get_file_location(pathformat) == \
+        (None, r'C:\Documents and Settings\Foobar\stuff.txt')
+
+    url = urls.url_parse('file://///server.tld/file.txt')
+    assert url.get_file_location(pathformat) == ('server.tld', r'file.txt')
+
+    url = urls.url_parse('file://///server.tld')
+    assert url.get_file_location(pathformat) == ('server.tld', '')
+
+    url = urls.url_parse('file://///%s' % localhost)
+    assert url.get_file_location(pathformat) == (None, '')
+
+    url = urls.url_parse('file://///%s/file.txt' % localhost)
+    assert url.get_file_location(pathformat) == (None, r'file.txt')
+
+
 def test_replace():
     url = urls.url_parse('http://de.wikipedia.org/wiki/Troll')
     strict_eq(url.replace(query='foo=bar'),
@@ -124,6 +164,25 @@ def test_url_fixing():
     x = urls.url_fix("http://just.a.test/$-_.+!*'(),")
     assert x == "http://just.a.test/$-_.+!*'(),"
 
+    x = urls.url_fix('http://höhöhö.at/höhöhö/hähähä')
+    assert x == r'http://xn--hhh-snabb.at/h%C3%B6h%C3%B6h%C3%B6/h%C3%A4h%C3%A4h%C3%A4'
+
+
+def test_url_fixing_filepaths():
+    x = urls.url_fix(r'file://C:\Users\Administrator\My Documents\ÑÈáÇíí')
+    assert x == r'file:///C%3A/Users/Administrator/My%20Documents/%C3%91%C3%88%C3%A1%C3%87%C3%AD%C3%AD'
+
+    a = urls.url_fix(r'file:/C:/')
+    b = urls.url_fix(r'file://C:/')
+    c = urls.url_fix(r'file:///C:/')
+    assert a == b == c == r'file:///C%3A/'
+
+    x = urls.url_fix(r'file://host/sub/path')
+    assert x == r'file://host/sub/path'
+
+    x = urls.url_fix(r'file:///')
+    assert x == r'file:///'
+
 
 def test_url_fixing_qs():
     x = urls.url_fix(b'http://example.com/?foo=%2f%2f')