Repository: pandas-dev/pandas

Issue:
BUG: python 3 compression and read_fwf
Getting a `TypeError: Type str doesn't support the buffer API` when using `read_fwf` on a compressed file with python 3 only; works in python 2.7

Example:

with a file `fwf.bug.txt` like

```
1111111111
2222222222
3333333333
4444444444

```

``` python
import pandas as pd

print(pd.__version__)

widths = [5, 5]

!gzip -d fwf_bug.txt

df = pd.read_fwf('fwf_bug.txt', widths=widths, names=['one', 'two'])
print(df)

!gzip fwf_bug.txt

# python 3 throws an error here.
df2 = pd.read_fwf('fwf_bug.txt.gz', widths=widths,
                  names=['one', 'two'], compression='gzip')

print(df2)
```

Versions:
python3: `0.11.1.dev-4d06037` should be most recent
python2: `0.11.1.dev-3ebfef9`

I can paste the full traceback if you'd like.



Discussion:
Doing a bit of digging around, looks like it's a unicode thing.

The error is coming here, line 1919 in pandas/build/lib.macosx-10.8-x86_64-3.3/pandas/io/parsers.py:

``` python

    def next(self):
        line = next(self.f)
        # Note: 'colspecs' is a sequence of half-open intervals.
        return [line[fromm:to].strip(self.filler or ' ')
                for (fromm, to) in self.colspecs]
```

here, `line` is a bytecode string: 

``` python
ipdb> line
b'1111111111\n'
```

I'm not sure what the preferred way of dealing with this is, but

``` python

ipdb> line.decode('utf-8')[fromm:to].strip(' ')
'11111'
```

works.

should probably be

``` python
import pandas.core.common as com
com.pprint_thing(line[fromm:to]).strip(self.filler or ' ')
```

basically, the problem is that you can't mix `bytes` and `str` in Python 3:

``` python
In [12]: b'abcd'.strip('')
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-12-e487f65c9b72> in <module>()
----> 1 b'abcd'.strip('')

TypeError: Type str doesn't support the buffer API

In [13]: # Whereas it works with bytes

In [14]: b'adbc'.strip(b'')
Out[14]: b'adbc'

```


Resolving patch:
diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py
--- a/pandas/io/parsers.py
+++ b/pandas/io/parsers.py
@@ -14,6 +14,7 @@
 from pandas.core.frame import DataFrame
 import datetime
 import pandas.core.common as com
+from pandas.core.config import get_option
 from pandas import compat
 from pandas.io.date_converters import generic_parser
 from pandas.io.common import get_filepath_or_buffer
@@ -1921,11 +1922,14 @@ class FixedWidthReader(object):
     """
     A reader of fixed-width lines.
     """
-    def __init__(self, f, colspecs, filler, thousands=None):
+    def __init__(self, f, colspecs, filler, thousands=None, encoding=None):
         self.f = f
         self.colspecs = colspecs
         self.filler = filler  # Empty characters between fields.
         self.thousands = thousands
+        if encoding is None:
+            encoding = get_option('display.encoding')
+        self.encoding = encoding
 
         if not ( isinstance(colspecs, (tuple, list))):
             raise AssertionError()
@@ -1937,11 +1941,20 @@ def __init__(self, f, colspecs, filler, thousands=None):
                        isinstance(colspec[1], int) ):
                 raise AssertionError()
 
-    def next(self):
-        line = next(self.f)
-        # Note: 'colspecs' is a sequence of half-open intervals.
-        return [line[fromm:to].strip(self.filler or ' ')
-                for (fromm, to) in self.colspecs]
+    if compat.PY3:
+        def next(self):
+            line = next(self.f)
+            if isinstance(line, bytes):
+                line = line.decode(self.encoding)
+            # Note: 'colspecs' is a sequence of half-open intervals.
+            return [line[fromm:to].strip(self.filler or ' ')
+                    for (fromm, to) in self.colspecs]
+    else:
+        def next(self):
+            line = next(self.f)
+            # Note: 'colspecs' is a sequence of half-open intervals.
+            return [line[fromm:to].strip(self.filler or ' ')
+                    for (fromm, to) in self.colspecs]
 
     # Iterator protocol in Python 3 uses __next__()
     __next__ = next
@@ -1959,7 +1972,8 @@ def __init__(self, f, **kwds):
         PythonParser.__init__(self, f, **kwds)
 
     def _make_reader(self, f):
-        self.data = FixedWidthReader(f, self.colspecs, self.delimiter)
+        self.data = FixedWidthReader(f, self.colspecs, self.delimiter,
+                                     encoding=self.encoding)
 
 
 ##### deprecations in 0.12 #####


Test patch:
diff --git a/pandas/io/tests/test_parsers.py b/pandas/io/tests/test_parsers.py
--- a/pandas/io/tests/test_parsers.py
+++ b/pandas/io/tests/test_parsers.py
@@ -2028,6 +2028,31 @@ def test_fwf_regression(self):
             res = df.loc[:,c]
             self.assert_(len(res))
 
+    def test_fwf_compression(self):
+        try:
+            import gzip
+            import bz2
+        except ImportError:
+            raise nose.SkipTest("Need gzip and bz2 to run this test")
+
+        data = """1111111111
+        2222222222
+        3333333333""".strip()
+        widths = [5, 5]
+        names = ['one', 'two']
+        expected = read_fwf(StringIO(data), widths=widths, names=names)
+        if compat.PY3:
+            data = bytes(data, encoding='utf-8')
+        for comp_name, compresser in [('gzip', gzip.GzipFile),
+                                      ('bz2', bz2.BZ2File)]:
+            with tm.ensure_clean() as path:
+                tmp = compresser(path, mode='wb')
+                tmp.write(data)
+                tmp.close()
+                result = read_fwf(path, widths=widths, names=names,
+                                  compression=comp_name)
+                tm.assert_frame_equal(result, expected)
+
     def test_verbose_import(self):
         text = """a,b,c,d
 one,1,2,3