Repository: pandas-dev/pandas

Issue:
BUG: pd.read_csv() fails when usecols contains multibyte unicode values
See PR: #13233 and pandas/io/tests/parser/usecols.py `test_usecols_with_multibyte_unicode_characters`
#### Code Sample, a copy-pastable example if possible

``` python
import pandas as pd
from pandas.compat import StringIO

s = '''あああ,いい,ううう,ええええ
0.056674973,8,True,a
2.613230982,2,False,b
3.568935038,7,False,a
'''
data = {
    'あああ': {
        0: 0.056674972999999997,
        1: 2.6132309819999997,
        2: 3.5689350380000002
    },
    'いい': {0: 8, 1: 2, 2: 7}
}
expected = pd.DataFrame(data)

df = pd.read_csv(StringIO(s), usecols=[u'あああ', u'いい'])
```

```
>>> df = pd.read_csv(StringIO(s), usecols=[u'あああ', u'いい'])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "pandas/io/parsers.py", line 562, in parser_f
    return _read(filepath_or_buffer, kwds)
  File "pandas/io/parsers.py", line 315, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "pandas/io/parsers.py", line 645, in __init__
    self._make_engine(self.engine)
  File "pandas/io/parsers.py", line 799, in _make_engine
    self._engine = CParserWrapper(self.f, **self.options)
  File "pandas/io/parsers.py", line 1263, in __init__
    raise ValueError("Usecols do not match names.")
ValueError: Usecols do not match names.

>>> df = pd.read_csv(StringIO(s), usecols=[u'あああ', u'いい'], engine='python')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "pandas/io/parsers.py", line 562, in parser_f
    return _read(filepath_or_buffer, kwds)
  File "pandas/io/parsers.py", line 315, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "pandas/io/parsers.py", line 645, in __init__
    self._make_engine(self.engine)
  File "pandas/io/parsers.py", line 805, in _make_engine
    self._engine = klass(self.f, **self.options)
  File "pandas/io/parsers.py", line 1609, in __init__
    self.columns, self.num_original_columns = self._infer_columns()
  File "pandas/io/parsers.py", line 1905, in _infer_columns
    columns = self._handle_usecols(columns, columns[0])
  File "pandas/io/parsers.py", line 1958, in _handle_usecols
    col_indices.append(usecols_key.index(u))
ValueError: u'\u3042\u3042\u3042' is not in list
```
#### Expected Output

```
        あああ  いい
0  0.056675   8
1  2.613231   2
2  3.568935   7
```
#### output of `pd.show_versions()`
## INSTALLED VERSIONS

commit: 71f5139f14b38a1a3c1d9fd7d9219947399faec0
python: 2.7.11.final.0
python-bits: 64
OS: Darwin
OS-release: 15.3.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

pandas: 0.18.1+54.g71f5139.dirty
nose: 1.3.7
pip: 8.1.2
setuptools: 20.7.0
Cython: 0.24
numpy: 1.11.0
scipy: None
statsmodels: None
xarray: None
IPython: None
sphinx: None
patsy: None
dateutil: 2.5.3
pytz: 2016.4
blosc: None
bottleneck: None
tables: None
numexpr: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
httplib2: None
apiclient: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: None
boto: None
pandas_datareader: None



Discussion:
@hassanshamim : Also show that it fails for the Python engine too (just pass in `engine=python' to get the error message and copy/paste into your original issue).

@jreback : This is a Python 2.x issue now (the test is enabled for Python 3.x)

https://github.com/pandas-dev/pandas/blob/01e7872b0/pandas/tests/io/parser/test_usecols.py#L388-L411

I think I have a patch for this, but I'm wondering whether it's worthwhile to add compatibility, or just close this out because of how close we are to releasing `0.24.0`, the last Python 2.x-compatible release?
u can patch if u want (0.24.x will be python 2 compat) or can close if it’s not trivial 

Resolving patch:
diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py
--- a/pandas/io/parsers.py
+++ b/pandas/io/parsers.py
@@ -1296,15 +1296,28 @@ def _validate_usecols_arg(usecols):
     if usecols is not None:
         if callable(usecols):
             return usecols, None
-        # GH20529, ensure is iterable container but not string.
-        elif not is_list_like(usecols):
+
+        if not is_list_like(usecols):
+            # see gh-20529
+            #
+            # Ensure it is iterable container but not string.
             raise ValueError(msg)
-        else:
-            usecols_dtype = lib.infer_dtype(usecols, skipna=False)
-            if usecols_dtype not in ('empty', 'integer',
-                                     'string', 'unicode'):
-                raise ValueError(msg)
-        return set(usecols), usecols_dtype
+
+        usecols_dtype = lib.infer_dtype(usecols, skipna=False)
+
+        if usecols_dtype not in ("empty", "integer",
+                                 "string", "unicode"):
+            raise ValueError(msg)
+
+        usecols = set(usecols)
+
+        if usecols_dtype == "unicode":
+            # see gh-13253
+            #
+            # Python 2.x compatibility
+            usecols = {col.encode("utf-8") for col in usecols}
+
+        return usecols, usecols_dtype
     return usecols, None
 
 


Test patch:
diff --git a/pandas/tests/io/parser/test_usecols.py b/pandas/tests/io/parser/test_usecols.py
--- a/pandas/tests/io/parser/test_usecols.py
+++ b/pandas/tests/io/parser/test_usecols.py
@@ -9,7 +9,7 @@
 import pytest
 
 from pandas._libs.tslib import Timestamp
-from pandas.compat import PY2, StringIO
+from pandas.compat import StringIO
 
 from pandas import DataFrame, Index
 import pandas.util.testing as tm
@@ -387,8 +387,7 @@ def test_usecols_with_mixed_encoding_strings(all_parsers, usecols):
 
 @pytest.mark.parametrize("usecols", [
     ["あああ", "いい"],
-    pytest.param([u"あああ", u"いい"], marks=pytest.mark.skipif(
-        PY2, reason="Buggy behavior: see gh-13253"))
+    [u"あああ", u"いい"]
 ])
 def test_usecols_with_multi_byte_characters(all_parsers, usecols):
     data = """あああ,いい,ううう,ええええ