Repository: frictionlessdata/tabulator-py
Pull request #147 (2017-01-24 11:27:40)

Implemented skipRows
# Overview

It will allow to skip arbitrary rows int the source like:
```python
with Stream(source, skip_rows=[1, 2, '#', '//'] as stream:
    print(stream.read()source)
```
In `goodtables.yml`:
```yml
files:
  - source: path
    skipRows: [1, 2, '#', '//']
```

roll: @amercader 
please take a look

Patch:
diff --git a/data/special/skip-rows.csv b/data/special/skip-rows.csv
new file mode 100644
index 0000000..6921c35
--- /dev/null
+++ b/data/special/skip-rows.csv
@@ -0,0 +1,4 @@
+id,name
+1,english
+# it's a comment!
+2,中国人
diff --git a/tabulator/stream.py b/tabulator/stream.py
index 7c109af..fdb3c8c 100644
--- a/tabulator/stream.py
+++ b/tabulator/stream.py
@@ -72,6 +72,11 @@ class Stream(object):
         post_parse (generator[]): post parse processors (hooks). Signature
             to follow is "processor(extended_rows)" which should yield
             one extended row (number, headers, row) per yield instruction.
+        skip_rows (int/str[]): list of rows to skip by:
+            - row number (add integers to the list)
+            - row comment (add strings to the list)
+            Example: skip_rows=[1, 2, '#', '//'] - rows 1, 2 and
+            all rows started with '#' and '//' will be skipped.
         options (dict): see in the scheme/format section
 
     """
@@ -86,6 +91,7 @@ class Stream(object):
                  encoding=None,
                  sample_size=100,
                  post_parse=[],
+                 skip_rows=[],
                  # DEPRECATED [v0.8-v1)
                  loader_options={},
                  parser_options={},
@@ -109,6 +115,15 @@ class Stream(object):
         elif isinstance(headers, int):
             self.__headers_row = headers
 
+        # Set skip rows
+        self.__skip_rows_by_numbers = []
+        self.__skip_rows_by_comments = []
+        for directive in copy(skip_rows):
+            if isinstance(directive, int):
+                self.__skip_rows_by_numbers.append(directive)
+            else:
+                self.__skip_rows_by_comments.append(str(directive))
+
         # Set attributes
         self.__source = source
         self.__scheme = scheme
@@ -377,11 +392,21 @@ class Stream(object):
 
     def __apply_processors(self, iterator):
 
-        # Apply processors to iterator
+        # Builtin processor
         def builtin_processor(extended_rows):
             for number, headers, row in extended_rows:
+                # Set headers
                 headers = self.__headers
+                # Skip row by numbers
+                if number in self.__skip_rows_by_numbers:
+                    continue
+                # Skip row by comments
+                match = lambda comment: row[0].startswith(comment)
+                if list(filter(match, self.__skip_rows_by_comments)):
+                    continue
                 yield (number, headers, row)
+
+        # Apply processors to iterator
         processors = [builtin_processor] + self.__post_parse
         for processor in processors:
             iterator = processor(iterator)


Test patch:
diff --git a/tests/test_stream.py b/tests/test_stream.py
index 7aee5b6..3e8f3a5 100644
--- a/tests/test_stream.py
+++ b/tests/test_stream.py
@@ -86,6 +86,21 @@ def test_stream_csv_doublequote():
             assert len(row) == 17
 
 
+# Tests [options]
+
+
+def test_stream_skip_rows():
+    source = 'data/special/skip-rows.csv'
+    with Stream(source, skip_rows=['#', 4]) as stream:
+        assert stream.read() == [['id', 'name'], ['1', 'english']]
+
+
+def test_stream_skip_rows_with_headers():
+    source = 'data/special/skip-rows.csv'
+    with Stream(source, headers=2, skip_rows=['#', 1]) as stream:
+        assert stream.read() == [['2', '中国人']]
+
+
 # Tests [options]
 
 def test_stream_csv_delimiter():