Repository: jazzband/tablib (python)

Issue:
Allow to skip first nth lines when reading a file
It would be nice to be able to skip the first lines when reading a file. The use case is especially frequent with Excel files, and when you want to read files without editing them first. Patch follows.

Pull request:
Fixes #496 - Allow skipping lines when reading xls/xlsx/csv files

Patch:
diff --git a/HISTORY.md b/HISTORY.md
index 0b28df9..cf25e01 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,13 @@
 # History
 
+## 3.1.0 (Unreleased)
+
+### Improvements
+
+- The csv, xls, and xlsx formats gained support for the `skip_lines` keyword
+  argument for their `import_set()` method to be able to skip the nth first
+  lines of a read file.
+
 ## 3.0.0 (2020-12-05)
 
 ### Breaking changes
diff --git a/docs/formats.rst b/docs/formats.rst
index 2357efe..c3620da 100644
--- a/docs/formats.rst
+++ b/docs/formats.rst
@@ -40,6 +40,13 @@ is headers with the ``headers`` boolean parameter (defaults to ``True``)::
 
     tablib.import_set(your_data_stream, format='csv', headers=False)
 
+It is also possible to provide the ``skip_lines`` parameter for the number of
+lines that should be skipped before starting to read data.
+
+.. versionchanged:: 3.1.0
+
+    The ``skip_lines`` parameter was added.
+
 When exporting with the ``csv`` format, the top row will contain headers, if
 they have been set. Otherwise, the top row will contain the first row of the
 dataset.
@@ -186,6 +193,14 @@ Import/export data in Legacy Excel Spreadsheet representation.
 This format is optional, install Tablib with ``pip install "tablib[xls]"`` to
 make the format available.
 
+Its ``import_set()`` method also supports a ``skip_lines`` parameter that you
+can set to a number of lines that should be skipped before starting to read
+data.
+
+.. versionchanged:: 3.1.0
+
+    The ``skip_lines`` parameter for ``import_set()`` was added.
+
 .. note::
 
     XLS files are limited to a maximum of 65,000 rows. Use xlsx_ to avoid this
@@ -215,6 +230,14 @@ programs generate XLSX files with incorrect dimensions.  Such files
 may need to be loaded with this optimization turned off by passing
 ``read_only=False``.
 
+The ``import_set()`` method also supports a ``skip_lines`` parameter that you
+can set to a number of lines that should be skipped before starting to read
+data.
+
+.. versionchanged:: 3.1.0
+
+    The ``skip_lines`` parameter for ``import_set()`` was added.
+
 .. note::
 
     When reading an ``xlsx`` file containing formulas in its cells, Tablib will
diff --git a/src/tablib/formats/_csv.py b/src/tablib/formats/_csv.py
index b0bf435..80676bb 100644
--- a/src/tablib/formats/_csv.py
+++ b/src/tablib/formats/_csv.py
@@ -33,7 +33,7 @@ class CSVFormat:
         return stream.getvalue()
 
     @classmethod
-    def import_set(cls, dset, in_stream, headers=True, **kwargs):
+    def import_set(cls, dset, in_stream, headers=True, skip_lines=0, **kwargs):
         """Returns dataset from CSV stream."""
 
         dset.wipe()
@@ -42,8 +42,9 @@ class CSVFormat:
 
         rows = csv.reader(in_stream, **kwargs)
         for i, row in enumerate(rows):
-
-            if (i == 0) and (headers):
+            if i < skip_lines:
+                continue
+            if i == skip_lines and headers:
                 dset.headers = row
             elif row:
                 if i > 0 and len(row) < dset.width:
diff --git a/src/tablib/formats/_xls.py b/src/tablib/formats/_xls.py
index 609810e..4a9e881 100644
--- a/src/tablib/formats/_xls.py
+++ b/src/tablib/formats/_xls.py
@@ -66,7 +66,7 @@ class XLSFormat:
         return stream.getvalue()
 
     @classmethod
-    def import_set(cls, dset, in_stream, headers=True):
+    def import_set(cls, dset, in_stream, headers=True, skip_lines=0):
         """Returns databook from XLS stream."""
 
         dset.wipe()
@@ -84,8 +84,10 @@ class XLSFormat:
             return value
 
         for i in range(sheet.nrows):
-            if i == 0 and headers:
-                dset.headers = sheet.row_values(0)
+            if i < skip_lines:
+                continue
+            if i == skip_lines and headers:
+                dset.headers = sheet.row_values(i)
             else:
                 dset.append([
                     cell_value(val, typ)
diff --git a/src/tablib/formats/_xlsx.py b/src/tablib/formats/_xlsx.py
index a6cb158..888b969 100644
--- a/src/tablib/formats/_xlsx.py
+++ b/src/tablib/formats/_xlsx.py
@@ -74,7 +74,7 @@ class XLSXFormat:
         return stream.getvalue()
 
     @classmethod
-    def import_set(cls, dset, in_stream, headers=True, read_only=True):
+    def import_set(cls, dset, in_stream, headers=True, read_only=True, skip_lines=0):
         """Returns databook from XLS stream."""
 
         dset.wipe()
@@ -85,8 +85,10 @@ class XLSXFormat:
         dset.title = sheet.title
 
         for i, row in enumerate(sheet.rows):
+            if i < skip_lines:
+                continue
             row_vals = [c.value for c in row]
-            if (i == 0) and (headers):
+            if i == skip_lines and headers:
                 dset.headers = row_vals
             else:
                 dset.append(row_vals)


Test patch:
diff --git a/tests/test_tablib.py b/tests/test_tablib.py
index e89eb0c..e9b10fa 100755
--- a/tests/test_tablib.py
+++ b/tests/test_tablib.py
@@ -814,6 +814,16 @@ class CSVTests(BaseTestCase):
             'F |  |  '
         )
 
+    def test_csv_import_set_skip_lines(self):
+        csv_text = (
+            'garbage,line\n'
+            '\n'
+            'id,name,description\n'
+            '12,Smith,rounded\n'
+        )
+        dataset = tablib.import_set(csv_text, format="csv", skip_lines=2)
+        self.assertEqual(dataset.headers, ['id', 'name','description'])
+
     def test_csv_export(self):
         """Verify exporting dataset object as CSV."""
 
@@ -984,6 +994,14 @@ class XLSTests(BaseTestCase):
             dset = tablib.Dataset().load(fh, 'xls')
         self.assertEqual(dset.dict[0]['birth_date'], datetime.datetime(2015, 4, 12, 0, 0))
 
+    def test_xlsx_import_set_skip_lines(self):
+        data.append(('garbage', 'line', ''))
+        data.append(('', '', ''))
+        data.append(('id', 'name', 'description'))
+        _xls = data.xls
+        new_data = tablib.Dataset().load(_xls, skip_lines=2)
+        self.assertEqual(new_data.headers, ['id', 'name', 'description'])
+
     def test_xls_import_with_errors(self):
         """Errors from imported files are kept as errors."""
         xls_source = Path(__file__).parent / 'files' / 'errors.xls'
@@ -1018,6 +1036,14 @@ class XLSXTests(BaseTestCase):
         self.assertEqual(data.dict[0]['float'], 21.55)
         self.assertEqual(data.dict[0]['date/time'], date_time)
 
+    def test_xlsx_import_set_skip_lines(self):
+        data.append(('garbage', 'line', ''))
+        data.append(('', '', ''))
+        data.append(('id', 'name', 'description'))
+        _xlsx = data.xlsx
+        new_data = tablib.Dataset().load(_xlsx, skip_lines=2)
+        self.assertEqual(new_data.headers, ['id', 'name', 'description'])
+
     def test_xlsx_bad_chars_sheet_name(self):
         data.title = "this / is / good"
         _xlsx = data.xlsx
@@ -1052,6 +1078,7 @@ class XLSXTests(BaseTestCase):
             data = tablib.Dataset().load(fh, read_only=False)
         self.assertEqual(data.height, 3)
 
+
 class JSONTests(BaseTestCase):
     def test_json_format_detect(self):
         """Test JSON format detection."""