Repository: pandas-dev/pandas

Issue:
read_fwf 'infer' where first hundred lines differ from other lines
#### Code Sample, a copy-pastable example if possible

```
  1     1   -13.120080   0.229   0.484  -0.378  -0.872
  1     2    -1.902843  -0.090   0.256   1.791   0.967
  1     3   -22.050698  -0.176  -0.394   0.922  -0.454
  1     4   -30.349928   0.081  -0.194  -0.327  -0.981
  1     5   -22.204160  -0.168  -0.197   0.984  -0.266
  1     6   -28.001753  -0.065   0.597  -0.203  -0.802
  1     7   -17.247524   0.108   0.194   0.474   0.774
  1     8   -28.014811   0.017   0.994   0.493   0.112
  1     9   -13.325491   0.259   0.189  -1.275   0.149
  1    10   -10.063621   0.327   0.108  -1.784   0.061
...
115    18     5.697000   0.391  -0.027   0.252   1.000
115    19     8.324000  -0.283   0.132   0.227  -0.216
115    20    48.451000   0.070  -0.041   0.379  -0.082
115    21     0.146000   0.677   0.031  -0.561  -0.149
115    22     1.443000  -0.706  -0.033  -0.222   0.035
115    23     4.595000   0.654  -0.081   0.774   0.997
115    24     0.146000  -0.677   0.031   0.561  -0.149
115    25     4.595000   0.654  -0.081   0.774   0.997
115    26     6.769000  -0.363  -0.093  -0.298   0.996
115    27    24.157000  -0.280  -0.324  -0.142  -0.946
```
#### Problem description

I have a long [fixed-width file](https://paste.ubuntu.com/23806889/) (>100k lines) that whose head and tail are shown above. I want to read this file with pandas. I figure `pd.read_fwf` is the way to do this. The issue comes up because it reads the first hundred lines, which start with  `'  1'` to say "lets start reading at [2]" whereas the last hundred lines start with `115`, so it skips the initial `11` and starts the line with `5`, so I lose data.

A couple of approaches to solving this issue come to mind, though I'm sure there are others:

- Don't infer until all lines are scanned
- Take as an argument the number of lines to be scanned before concluding the format, including the option to scan all (e.g. `infer_from_all`)
- Take as an argument which direction to scan -- top to bottom or bottom to top


#### Output of ``pd.show_versions()``



<details>

commit: None
python: 2.7.6.final.0
python-bits: 64
OS: Linux
OS-release: 3.13.0-107-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
LOCALE: None.None


</details>



Discussion:
I think it would be reasonable to pass down a new parameter, maybe ``infer_nrows`` with a default of 100 to do this. Then you can control the amount of inference you want. It could accept ``infer_nrows='all'`` if you want to infer from all the rows I suppose.

PR's would be welcome!
I am on it :)
Ran into the same issue. Had to sort the file to have larger numbers or top.

I'd suggest, at least for numeric columns, not to infer the left boundary for the first column but go with 0.

I'd rather see a very conservative approach where nothing gets stripped/truncated initially. Rather, we can trim strings later on, if needed.

Thanks!
@yakovkeselman contributions welcome! This is pretty straightforward to actually implement.

Resolving patch:
diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py
--- a/pandas/io/parsers.py
+++ b/pandas/io/parsers.py
@@ -501,6 +501,7 @@ def _read(filepath_or_buffer, kwds):
 
 _fwf_defaults = {
     'colspecs': 'infer',
+    'infer_nrows': 100,
     'widths': None,
 }
 
@@ -718,8 +719,8 @@ def parser_f(filepath_or_buffer,
                       )(read_table)
 
 
-def read_fwf(filepath_or_buffer, colspecs='infer',
-             widths=None, **kwds):
+def read_fwf(filepath_or_buffer, colspecs='infer', widths=None,
+             infer_nrows=100, **kwds):
 
     r"""
     Read a table of fixed-width formatted lines into DataFrame.
@@ -752,6 +753,11 @@ def read_fwf(filepath_or_buffer, colspecs='infer',
     widths : list of int, optional
         A list of field widths which can be used instead of 'colspecs' if
         the intervals are contiguous.
+    infer_nrows : int, default 100
+        The number of rows to consider when letting the parser determine the
+        `colspecs`.
+
+        .. versionadded:: 0.24.0
     **kwds : optional
         Optional keyword arguments can be passed to ``TextFileReader``.
 
@@ -786,6 +792,7 @@ def read_fwf(filepath_or_buffer, colspecs='infer',
             col += w
 
     kwds['colspecs'] = colspecs
+    kwds['infer_nrows'] = infer_nrows
     kwds['engine'] = 'python-fwf'
     return _read(filepath_or_buffer, kwds)
 
@@ -3442,13 +3449,15 @@ class FixedWidthReader(BaseIterator):
     A reader of fixed-width lines.
     """
 
-    def __init__(self, f, colspecs, delimiter, comment, skiprows=None):
+    def __init__(self, f, colspecs, delimiter, comment, skiprows=None,
+                 infer_nrows=100):
         self.f = f
         self.buffer = None
         self.delimiter = '\r\n' + delimiter if delimiter else '\n\r\t '
         self.comment = comment
         if colspecs == 'infer':
-            self.colspecs = self.detect_colspecs(skiprows=skiprows)
+            self.colspecs = self.detect_colspecs(infer_nrows=infer_nrows,
+                                                 skiprows=skiprows)
         else:
             self.colspecs = colspecs
 
@@ -3464,19 +3473,20 @@ def __init__(self, f, colspecs, delimiter, comment, skiprows=None):
                 raise TypeError('Each column specification must be '
                                 '2 element tuple or list of integers')
 
-    def get_rows(self, n, skiprows=None):
+    def get_rows(self, infer_nrows, skiprows=None):
         """
         Read rows from self.f, skipping as specified.
 
-        We distinguish buffer_rows (the first <= n lines)
-        from the rows returned to detect_colspecs because
-        it's simpler to leave the other locations with
-        skiprows logic alone than to modify them to deal
-        with the fact we skipped some rows here as well.
+        We distinguish buffer_rows (the first <= infer_nrows
+        lines) from the rows returned to detect_colspecs
+        because it's simpler to leave the other locations
+        with skiprows logic alone than to modify them to
+        deal with the fact we skipped some rows here as
+        well.
 
         Parameters
         ----------
-        n : int
+        infer_nrows : int
             Number of rows to read from self.f, not counting
             rows that are skipped.
         skiprows: set, optional
@@ -3496,16 +3506,16 @@ def get_rows(self, n, skiprows=None):
             if i not in skiprows:
                 detect_rows.append(row)
             buffer_rows.append(row)
-            if len(detect_rows) >= n:
+            if len(detect_rows) >= infer_nrows:
                 break
         self.buffer = iter(buffer_rows)
         return detect_rows
 
-    def detect_colspecs(self, n=100, skiprows=None):
+    def detect_colspecs(self, infer_nrows=100, skiprows=None):
         # Regex escape the delimiters
         delimiters = ''.join(r'\%s' % x for x in self.delimiter)
         pattern = re.compile('([^%s]+)' % delimiters)
-        rows = self.get_rows(n, skiprows)
+        rows = self.get_rows(infer_nrows, skiprows)
         if not rows:
             raise EmptyDataError("No rows from which to infer column width")
         max_len = max(map(len, rows))
@@ -3544,8 +3554,10 @@ class FixedWidthFieldParser(PythonParser):
     def __init__(self, f, **kwds):
         # Support iterators, convert to a list.
         self.colspecs = kwds.pop('colspecs')
+        self.infer_nrows = kwds.pop('infer_nrows')
         PythonParser.__init__(self, f, **kwds)
 
     def _make_reader(self, f):
         self.data = FixedWidthReader(f, self.colspecs, self.delimiter,
-                                     self.comment, self.skiprows)
+                                     self.comment, self.skiprows,
+                                     self.infer_nrows)


Test patch:
diff --git a/pandas/tests/io/parser/test_read_fwf.py b/pandas/tests/io/parser/test_read_fwf.py
--- a/pandas/tests/io/parser/test_read_fwf.py
+++ b/pandas/tests/io/parser/test_read_fwf.py
@@ -140,6 +140,22 @@ def test_fwf_colspecs_None(self):
         expected = DataFrame([[123456, 456], [456789, 789]])
         tm.assert_frame_equal(result, expected)
 
+    def test_fwf_colspecs_infer_nrows(self):
+        # GH 15138
+        data = """\
+  1  2
+123 98
+"""
+        # infer_nrows == 1 should have colspec == [(2, 3), (5, 6)]
+        df = read_fwf(StringIO(data), header=None, infer_nrows=1)
+        expected = pd.DataFrame([[1, 2], [3, 8]])
+        tm.assert_frame_equal(df, expected)
+
+        # test for infer_nrows > number of rows
+        df = read_fwf(StringIO(data), header=None, infer_nrows=10)
+        expected = pd.DataFrame([[1, 2], [123, 98]])
+        tm.assert_frame_equal(df, expected)
+
     def test_fwf_regression(self):
         # GH 3594
         # turns out 'T060' is parsable as a datetime slice!