Repository: dask/dask

Issue:
infer_storage_options lower-cases bucket names
Because `dask.utils.infer_storage_options` interprets a path as a generic URL, the bucket part of a path like "s3://mybucket/loc/mykey" becomes the host. Actual hosts are always lower-case, so the consequence is that only a lower-case version of the bucket name is kept. Thus, buckets with upper-case letters cannot be found.

@frol , do you have a moment to consider this?



Discussion:
I've also run into some trouble with `infer_storage_options` and Windows paths, which often start at `C:\...`, so that `C:` gets mistakenly read as a protocol.  I special cased it in `infer_storage_options` to solve the problem short term, but as you point out it's clear that the URL handling makes some inaccurate assumptions.  On the other hand the url handling is quite convenient for picking out username/password combinations and such.  I don't know what the right course of action is.


Resolving patch:
diff --git a/dask/utils.py b/dask/utils.py
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -6,6 +6,7 @@
 import functools
 import io
 import os
+import re
 import sys
 import shutil
 import struct
@@ -702,7 +703,8 @@ def infer_storage_options(urlpath, inherit_storage_options=None):
     "host": "node", "port": 123, "path": "/mnt/datasets/test.csv",
     "url_query": "q=1", "extra": "value"}
     """
-    if ':\\' in urlpath:
+    # Handle Windows paths including disk name in this special case
+    if re.match(r'^[a-zA-Z]:\\', urlpath):
         return {'protocol': 'file',
                 'path': urlpath}
 
@@ -714,7 +716,10 @@ def infer_storage_options(urlpath, inherit_storage_options=None):
     }
 
     if parsed_path.netloc:
-        inferred_storage_options['host'] = parsed_path.hostname
+        # Parse `hostname` from netloc manually because `parsed_path.hostname`
+        # lowercases the hostname which is not always desirable (e.g. in S3):
+        # https://github.com/dask/dask/issues/1417
+        inferred_storage_options['host'] = parsed_path.netloc.rsplit('@', 1)[-1].rsplit(':', 1)[0]
         if parsed_path.port:
             inferred_storage_options['port'] = parsed_path.port
         if parsed_path.username:


Test patch:
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -132,13 +132,23 @@ def test_infer_storage_options():
     assert infer_storage_options('./test.csv')['path'] == './test.csv'
     assert infer_storage_options('../test.csv')['path'] == '../test.csv'
 
+    so = infer_storage_options('C:\\test.csv')
+    assert so.pop('protocol') == 'file'
+    assert so.pop('path') == 'C:\\test.csv'
+    assert not so
+
+    assert infer_storage_options('d:\\test.csv')['path'] == 'd:\\test.csv'
+    assert infer_storage_options('\\test.csv')['path'] == '\\test.csv'
+    assert infer_storage_options('.\\test.csv')['path'] == '.\\test.csv'
+    assert infer_storage_options('test.csv')['path'] == 'test.csv'
+
     so = infer_storage_options(
-              'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1#fragm',
+              'hdfs://username:pwd@Node:123/mnt/datasets/test.csv?q=1#fragm',
               inherit_storage_options={'extra': 'value'})
     assert so.pop('protocol') == 'hdfs'
     assert so.pop('username') == 'username'
     assert so.pop('password') == 'pwd'
-    assert so.pop('host') == 'node'
+    assert so.pop('host') == 'Node'
     assert so.pop('port') == 123
     assert so.pop('path') == '/mnt/datasets/test.csv'
     assert so.pop('url_query') == 'q=1'
@@ -146,6 +156,13 @@ def test_infer_storage_options():
     assert so.pop('extra') == 'value'
     assert not so
 
+    so = infer_storage_options('hdfs://User-name@Node-name.com/mnt/datasets/test.csv')
+    assert so.pop('username') == 'User-name'
+    assert so.pop('host') == 'Node-name.com'
+
+    assert infer_storage_options('s3://Bucket-name.com/test.csv')['host'] == 'Bucket-name.com'
+    assert infer_storage_options('http://127.0.0.1:8080/test.csv')['host'] == '127.0.0.1'
+
     with pytest.raises(KeyError):
         infer_storage_options('file:///bucket/file.csv', {'path': 'collide'})
     with pytest.raises(KeyError):