Repository: pypa/pip

Issue:
Error running pip install -r <requirements file>
Python 3.4 / Windows 7 / Powershell / pip version 7.0.3

Running 

```
pip install -r .\stable-req.txt
```

with stable-req.txt containing requests==2.7.0

results in the following exception:

```
Traceback (most recent call last):
  File "D:\frh\env2\lib\site-packages\pip\basecommand.py", line 223, in main
    status = self.run(options, args)
  File "D:\frh\env2\lib\site-packages\pip\commands\install.py", line 268, in run
    wheel_cache
  File "D:\frh\env2\lib\site-packages\pip\basecommand.py", line 287, in populate_requirement_set
    wheel_cache=wheel_cache):
  File "D:\frh\env2\lib\site-packages\pip\req\req_file.py", line 86, in parse_requirements
    for req in req_iter:
  File "D:\frh\env2\lib\site-packages\pip\req\req_file.py", line 130, in process_line
    wheel_cache=wheel_cache
  File "D:\frh\env2\lib\site-packages\pip\req\req_install.py", line 207, in from_line
    wheel_cache=wheel_cache)
  File "D:\frh\env2\lib\site-packages\pip\req\req_install.py", line 66, in __init__
    req = pkg_resources.Requirement.parse(req)
  File "D:\frh\env2\lib\site-packages\pip\_vendor\pkg_resources\__init__.py", line 2960, in parse
    reqs = list(parse_requirements(s))
  File "D:\frh\env2\lib\site-packages\pip\_vendor\pkg_resources\__init__.py", line 2904, in parse_requirements
    "version spec")
  File "D:\frh\env2\lib\site-packages\pip\_vendor\pkg_resources\__init__.py", line 2869, in scan_list
    raise ValueError(msg, line, "at", line[p:])
ValueError: ('Expected version spec in', 'ÿþr\x00e\x00q\x00u\x00e\x00s\x00t\x00s\x00=\x00=\x002\x00.\x007\x00.\x000\x00'
, 'at', '\x00e\x00q\x00u\x00e\x00s\x00t\x00s\x00=\x00=\x002\x00.\x007\x00.\x000\x00')
```

Reproduced on two pcs and one Windows Server 2008 r2



Discussion:
looks like a big pile of encoding confusion. Is that UCS32 perhaps?

```
>>> u"ÿþr\x00e\x00q\x00u\x00e\x00s\x00t\x00s\x00=\x00=\x002\x00.\x007\x00.\x000\x00".encode('utf8')
b'\xc3\xbf\xc3\xber\x00e\x00q\x00u\x00e\x00s\x00t\x00s\x00=\x00=\x002\x00.\x007\x00.\x000\x00'
>>> u"ÿþr\x00e\x00q\x00u\x00e\x00s\x00t\x00s\x00=\x00=\x002\x00.\x007\x00.\x000\x00".encode('utf8').decode('utf16')
'뿃뻃requests==2.7.0'
```

and `ÿþ` seems to corresponds to the UTF-16 (LE) BOM (cf https://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding), which would match.

So I'd guess the req file was encoded in utf16 but python opened it as utf8 (returned from `locale.getpreferredencoding()` if I understand correctly https://docs.python.org/3.4/library/functions.html#open).

Not sure what pip could do to improve the user experience on such case ...

Yeah, that's a UTF-16 file. Was it created by redirection in Powershell (`some_cmd > stable-req.txt`) ? As that is typically where I end up with UTF-16 files that I didn't want ;-)

Auto-detecting (guessing) encodings is generally a bad idea, and probably something pip shouldn't get into, but I guess that looking for a BOM when reading the requirements file, and choosing the encoding from that if it's found, would be a useful feature to help with this common Windows issue. (Although it's mainly a Windows issue, the BOM detection can be added for all platforms, as I can't see how it would cause a problem on any other platform).

Duplicate of #733


Resolving patch:
diff --git a/pip/download.py b/pip/download.py
--- a/pip/download.py
+++ b/pip/download.py
@@ -29,6 +29,7 @@
 from pip.utils import (splitext, rmtree, format_size, display_path,
                        backup_dir, ask_path_exists, unpack_file,
                        ARCHIVE_EXTENSIONS, consume, call_subprocess)
+from pip.utils.encoding import auto_decode
 from pip.utils.filesystem import check_path_owner
 from pip.utils.logging import indent_log
 from pip.utils.setuptools_build import SETUPTOOLS_SHIM
@@ -407,14 +408,10 @@ def get_file_content(url, comes_from=None, session=None):
             # FIXME: catch some errors
             resp = session.get(url)
             resp.raise_for_status()
-
-            if six.PY3:
-                return resp.url, resp.text
-            else:
-                return resp.url, resp.content
+            return resp.url, resp.text
     try:
-        with open(url) as f:
-            content = f.read()
+        with open(url, 'rb') as f:
+            content = auto_decode(f.read())
     except IOError as exc:
         raise InstallationError(
             'Could not open requirements file: %s' % str(exc)
diff --git a/pip/req/req_file.py b/pip/req/req_file.py
--- a/pip/req/req_file.py
+++ b/pip/req/req_file.py
@@ -7,6 +7,7 @@
 import os
 import re
 import shlex
+import sys
 import optparse
 import warnings
 
@@ -133,6 +134,9 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
         # `finder.format_control` will be updated during parsing
         defaults.format_control = finder.format_control
     args_str, options_str = break_args_options(line)
+    if sys.version_info < (2, 7, 3):
+        # Priori to 2.7.3, shlex can not deal with unicode entries
+        options_str = options_str.encode('utf8')
     opts, _ = parser.parse_args(shlex.split(options_str), defaults)
 
     # preserve for the nested code path
diff --git a/pip/utils/encoding.py b/pip/utils/encoding.py
new file mode 100644
--- /dev/null
+++ b/pip/utils/encoding.py
@@ -0,0 +1,23 @@
+import codecs
+import locale
+
+
+BOMS = [
+    (codecs.BOM_UTF8, 'utf8'),
+    (codecs.BOM_UTF16, 'utf16'),
+    (codecs.BOM_UTF16_BE, 'utf16-be'),
+    (codecs.BOM_UTF16_LE, 'utf16-le'),
+    (codecs.BOM_UTF32, 'utf32'),
+    (codecs.BOM_UTF32_BE, 'utf32-be'),
+    (codecs.BOM_UTF32_LE, 'utf32-le'),
+]
+
+
+def auto_decode(data):
+    """Check a bytes string for a BOM to correctly detect the encoding
+
+    Fallback to locale.getpreferredencoding(False) like open() on Python3"""
+    for bom, encoding in BOMS:
+        if data.startswith(bom):
+            return data[len(bom):].decode(encoding)
+    return data.decode(locale.getpreferredencoding(False))


Test patch:
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 """
 util tests
 
@@ -15,6 +17,7 @@
 from pip.exceptions import HashMismatch, HashMissing, InstallationError
 from pip.utils import (egg_link_path, get_installed_distributions,
                        untar_file, unzip_file, rmtree, normalize_path)
+from pip.utils.encoding import auto_decode
 from pip.utils.hashes import Hashes, MissingHashes
 from pip._vendor.six import BytesIO
 
@@ -447,3 +450,17 @@ def test_non_zero(self):
         assert Hashes({'sha256': 'dummy'})
         assert not Hashes()
         assert not Hashes({})
+
+
+class TestEncoding(object):
+    """Tests for pip.utils.encoding"""
+
+    def test_auto_decode_utf16_le(self):
+        data = (
+            b'\xff\xfeD\x00j\x00a\x00n\x00g\x00o\x00=\x00'
+            b'=\x001\x00.\x004\x00.\x002\x00'
+        )
+        assert auto_decode(data) == "Django==1.4.2"
+
+    def test_auto_decode_no_bom(self):
+        assert auto_decode(b'foobar') == u'foobar'