Repository: pandas-dev/pandas

Issue:
Issue iterating with pd.read_sas
#### Code Sample, a copy-pastable example if possible

```
import pandas as pd

f = pd.read_sas('test.sas7bdat', iterator=True)
for chunk in f:
    print(f)

---------------------------------------------------------------------------
AbstractMethodError                       Traceback (most recent call last)
<ipython-input-3-bedf2769bffe> in <module>()
----> 1 for chunk in f:
      2     print(chunk)

C:\Anaconda3\lib\site-packages\pandas\io\common.py in __next__(self)
     99 
    100     def __next__(self):
--> 101         raise AbstractMethodError(self)
    102 
    103 if not compat.PY3:

AbstractMethodError: This method must be defined in the concrete class of SAS7BDATReader
```
#### Expected Output

Print of each chunk

I'm no expert, but it seems as if the SAS7BDAT class inherits from BaseIterator but does not provide its own `__next__` method
#### output of `pd.show_versions()`
## INSTALLED VERSIONS

commit: None
python: 3.5.1.final.0
python-bits: 64
OS: Windows
OS-release: 7
machine: AMD64
processor: Intel64 Family 6 Model 60 Stepping 3, GenuineIntel
byteorder: little
LC_ALL: None
LANG: None

pandas: 0.18.1
nose: 1.3.7
pip: 8.1.2
setuptools: 20.3
Cython: 0.23.4
numpy: 1.10.4
scipy: 0.17.0
statsmodels: 0.6.1
xarray: None
IPython: 4.1.2
sphinx: 1.3.1
patsy: 0.4.1
dateutil: 2.5.1
pytz: 2016.2
blosc: None
bottleneck: 1.0.0
tables: 3.2.2
numexpr: 2.5.2
matplotlib: 1.5.1
openpyxl: 2.3.2
xlrd: 0.9.4
xlwt: 1.0.0
xlsxwriter: 0.8.4
lxml: 3.6.0
bs4: 4.4.1
html5lib: None
httplib2: None
apiclient: None
sqlalchemy: 1.0.12
pymysql: None
psycopg2: None
jinja2: 2.8
boto: 2.39.0
pandas_datareader: None



Discussion:
@andersbogsnes That seems like a correct observation! Do you want to make a PR for this? Shouldn't be too difficult, you can see how it is done for the xport reader: https://github.com/pydata/pandas/blob/master/pandas/io/sas/sas_xport.py#L354

Sure, I'll give it a go, seems doable!


Resolving patch:
diff --git a/pandas/io/sas/sas7bdat.py b/pandas/io/sas/sas7bdat.py
--- a/pandas/io/sas/sas7bdat.py
+++ b/pandas/io/sas/sas7bdat.py
@@ -225,6 +225,12 @@ def _get_properties(self):
                 self.os_name = self.os_name.decode(
                     self.encoding or self.default_encoding)
 
+    def __next__(self):
+        da = self.read(nrows=self.chunksize or 1)
+        if da is None:
+            raise StopIteration
+        return da
+
     # Read a single float of the given width (4 or 8).
     def _read_float(self, offset, width):
         if width not in (4, 8):
@@ -591,6 +597,10 @@ def read(self, nrows=None):
         if self._current_row_in_file_index >= self.row_count:
             return None
 
+        m = self.row_count - self._current_row_in_file_index
+        if nrows > m:
+            nrows = m
+
         nd = (self.column_types == b'd').sum()
         ns = (self.column_types == b's').sum()
 


Test patch:
diff --git a/pandas/io/tests/sas/test_sas7bdat.py b/pandas/io/tests/sas/test_sas7bdat.py
--- a/pandas/io/tests/sas/test_sas7bdat.py
+++ b/pandas/io/tests/sas/test_sas7bdat.py
@@ -47,7 +47,9 @@ def test_from_buffer(self):
                 with open(fname, 'rb') as f:
                     byts = f.read()
                 buf = io.BytesIO(byts)
-                df = pd.read_sas(buf, format="sas7bdat", encoding='utf-8')
+                rdr = pd.read_sas(buf, format="sas7bdat",
+                                  iterator=True, encoding='utf-8')
+                df = rdr.read()
                 tm.assert_frame_equal(df, df0, check_exact=False)
 
     def test_from_iterator(self):
@@ -55,16 +57,35 @@ def test_from_iterator(self):
             df0 = self.data[j]
             for k in self.test_ix[j]:
                 fname = os.path.join(self.dirpath, "test%d.sas7bdat" % k)
-                with open(fname, 'rb') as f:
-                    byts = f.read()
-                buf = io.BytesIO(byts)
-                rdr = pd.read_sas(buf, format="sas7bdat",
-                                  iterator=True, encoding='utf-8')
+                rdr = pd.read_sas(fname, iterator=True, encoding='utf-8')
                 df = rdr.read(2)
                 tm.assert_frame_equal(df, df0.iloc[0:2, :])
                 df = rdr.read(3)
                 tm.assert_frame_equal(df, df0.iloc[2:5, :])
 
+    def test_iterator_loop(self):
+        # github #13654
+        for j in 0, 1:
+            for k in self.test_ix[j]:
+                for chunksize in 3, 5, 10, 11:
+                    fname = os.path.join(self.dirpath, "test%d.sas7bdat" % k)
+                    rdr = pd.read_sas(fname, chunksize=10, encoding='utf-8')
+                    y = 0
+                    for x in rdr:
+                        y += x.shape[0]
+                    self.assertTrue(y == rdr.row_count)
+
+    def test_iterator_read_too_much(self):
+        # github #14734
+        k = self.test_ix[0][0]
+        fname = os.path.join(self.dirpath, "test%d.sas7bdat" % k)
+        rdr = pd.read_sas(fname, format="sas7bdat",
+                          iterator=True, encoding='utf-8')
+        d1 = rdr.read(rdr.row_count + 20)
+        rdr = pd.read_sas(fname, iterator=True, encoding="utf-8")
+        d2 = rdr.read(rdr.row_count + 20)
+        tm.assert_frame_equal(d1, d2)
+
 
 def test_encoding_options():
     dirpath = tm.get_data_path()
diff --git a/pandas/io/tests/sas/test_xport.py b/pandas/io/tests/sas/test_xport.py
--- a/pandas/io/tests/sas/test_xport.py
+++ b/pandas/io/tests/sas/test_xport.py
@@ -35,6 +35,13 @@ def test1_basic(self):
         # Read full file
         data = read_sas(self.file01, format="xport")
         tm.assert_frame_equal(data, data_csv)
+        num_rows = data.shape[0]
+
+        # Test reading beyond end of file
+        reader = read_sas(self.file01, format="xport", iterator=True)
+        data = reader.read(num_rows + 100)
+        self.assertTrue(data.shape[0] == num_rows)
+        reader.close()
 
         # Test incremental read with `read` method.
         reader = read_sas(self.file01, format="xport", iterator=True)
@@ -48,6 +55,14 @@ def test1_basic(self):
         reader.close()
         tm.assert_frame_equal(data, data_csv.iloc[0:10, :])
 
+        # Test read in loop
+        m = 0
+        reader = read_sas(self.file01, format="xport", chunksize=100)
+        for x in reader:
+            m += x.shape[0]
+        reader.close()
+        self.assertTrue(m == num_rows)
+
         # Read full file with `read_sas` method
         data = read_sas(self.file01)
         tm.assert_frame_equal(data, data_csv)