Repository: dask/dask

Issue:
skiprows in read_fwf: Link to Issue #2065
Four years ago in issue #2065 an issue with skiprows was raised. At that time dask.dataframe.read_csv would skip the number of rows specified in skiprows from every partition, not just the first partition. This was acknowledged as a bug and fixed in pull request #2165.
https://github.com/dask/dask/pull/2165

There have been significant changes in that part of the code base and it appears that this bug has reverted.

    import dask.dataframe as dd
    import pandas as pd
    import dask
    
    print(dask.__version__)
    #  '2021.04.1'
    df = pd.read_fwf('data.txt', colspecs=<colspecs>, skiprows=1)
    ddf = dd.read_fwf('data.txt', colspecs=<colspecs>, skiprows=1).compute()
    print(len(df))
    # 1000000
    print(len(ddf))
    # 9999974
    print(ddf.npartitions)
    # 27




Discussion:
Thanks for reporting @rhjmoore. I could be missing something, but I'm not able to reproduce with the latest `dask=2021.5.0` release with the snippet below:

```python
import pandas as pd
import dask.dataframe as dd

df = pd.DataFrame({"a": range(10_000)})
df.to_csv("foo.csv", index=False)

ddf = dd.read_fwf("foo.csv", colspecs="infer", blocksize=600, sample=600, skiprows=1)
print(f"{ddf.npartitions = }")   # Should have more than 1 partition
print(f"{len(df) = }")
print(f"{len(ddf) = }")
```

outputs

```
ddf.npartitions = 82
len(df) = 10000
len(ddf) = 9999
```

Note that there are multiple partitions for `ddf` and the number of rows is one less than the corresponding pandas DataFrame (which is consistent with `skiprows=1`).
Apologies, you are right. The bug this time is in skipfooter. I simplified my code too much and did not notice the other possibility.

There is a bug, but the bug is in skipfooter not in skiprows. Try this:

    df = pd.DataFrame({"a": range(10_000)})
    df.to_csv("foo.csv", index=False)

    ddf = dd.read_fwf("foo.csv", colspecs="infer", blocksize=600, sample=600, skiprows=1, skipfooter=1)
    print(f"{ddf.npartitions = }")   # Should have more than 1 partition
    print(f"{len(df) = }")
    print(f"{len(ddf) = }")

@rhjmoore Thank you for the example, I was able to reproduce the bug. It seems we will need to do a fix similar to the one in the PR #2165 . Would you like to take shot at it? 
> @rhjmoore Thank you for the example, I was able to reproduce the bug. It seems we will need to do a fix similar to the one in the PR #2165 . Would you like to take shot at it?

@ncclementi I could, yes, but have never contributed to such a project before. I'd just git clone, make a new branch from master, make the changes, then pull request (anyone in particular I should address it to?)?
Yeah, those are most of the main steps. Our complete contributing guide is available at https://docs.dask.org/en/latest/develop.html. Feel free to ping me and @ncclementi on the PR. Thanks! 
@jrbourbeau It looks like I'll have to change the structure of "parts" in dask.dataframe.io.csv line 372 to include a 4th entry named "is_last" and a new block_mask function to do the opposite of the existing one used for "is_first". Do you see a smarter way of knowing which block is the last one?
That seems like the right approach to me @rhjmoore. You might want to start by writing a test like the one in the PR that you linked https://github.com/dask/dask/pull/2165/files#diff-e417090a76d08168f0617f553826da07a1a190ec50b9f60fbe6462680224eab7

Resolving patch:
diff --git a/dask/dataframe/io/csv.py b/dask/dataframe/io/csv.py
--- a/dask/dataframe/io/csv.py
+++ b/dask/dataframe/io/csv.py
@@ -86,7 +86,7 @@ def project_columns(self, columns):
     def __call__(self, part):
 
         # Part will be a 3-element tuple
-        block, path, is_first = part
+        block, path, is_first, is_last = part
 
         # Construct `path_info`
         if path is not None:
@@ -105,6 +105,8 @@ def __call__(self, part):
         if not is_first:
             write_header = True
             rest_kwargs.pop("skiprows", None)
+        if not is_last:
+            rest_kwargs.pop("skipfooter", None)
 
         # Deal with column projection
         columns = self.full_columns
@@ -346,6 +348,7 @@ def text_blocks_to_pandas(
     blocks = tuple(flatten(block_lists))
     # Create mask of first blocks from nested block_lists
     is_first = tuple(block_mask(block_lists))
+    is_last = tuple(block_mask_last(block_lists))
 
     if path:
         colname, path_converter = path
@@ -368,13 +371,7 @@ def text_blocks_to_pandas(
     parts = []
     colname, paths = path or (None, None)
     for i in range(len(blocks)):
-        parts.append(
-            [
-                blocks[i],
-                paths[i] if paths else None,
-                is_first[i],
-            ]
-        )
+        parts.append([blocks[i], paths[i] if paths else None, is_first[i], is_last[i]])
 
     # Create Blockwise layer
     label = "read-csv-"
@@ -416,6 +413,21 @@ def block_mask(block_lists):
         yield from (False for _ in block[1:])
 
 
+def block_mask_last(block_lists):
+    """
+    Yields a flat iterable of booleans to mark the last element of the
+    nested input ``block_lists`` in a flattened output.
+
+    >>> list(block_mask_last([[1, 2], [3, 4], [5]]))
+    [False, True, False, True, True]
+    """
+    for block in block_lists:
+        if not block:
+            continue
+        yield from (False for _ in block[:-1])
+        yield True
+
+
 def auto_blocksize(total_memory, cpu_count):
     memory_factor = 10
     blocksize = int(total_memory // cpu_count / memory_factor)
@@ -576,8 +588,10 @@ def read_pandas(
     header = b"" if header is None else parts[firstrow] + b_lineterminator
 
     # Use sample to infer dtypes and check for presence of include_path_column
+    head_kwargs = kwargs.copy()
+    head_kwargs.pop("skipfooter", None)
     try:
-        head = reader(BytesIO(b_sample), nrows=sample_rows, **kwargs)
+        head = reader(BytesIO(b_sample), nrows=sample_rows, **head_kwargs)
     except pd.errors.ParserError as e:
         if "EOF" in str(e):
             raise ValueError(


Test patch:
diff --git a/dask/dataframe/io/tests/test_csv.py b/dask/dataframe/io/tests/test_csv.py
--- a/dask/dataframe/io/tests/test_csv.py
+++ b/dask/dataframe/io/tests/test_csv.py
@@ -124,6 +124,10 @@ def parse_filename(path):
 # in a data file
 # before any data"""
 
+comment_footer = b"""# some footer lines
+# that may be present
+# at the end of the file"""
+
 csv_units_row = b"str, int, int\n"
 tsv_units_row = csv_units_row.replace(b",", b"\t")
 
@@ -250,6 +254,22 @@ def test_skiprows(dd_read, pd_read, files):
         assert_eq(df, expected_df, check_dtype=False)
 
 
+@pytest.mark.parametrize(
+    "dd_read,pd_read,files",
+    [(dd.read_csv, pd.read_csv, csv_files), (dd.read_table, pd.read_table, tsv_files)],
+)
+@read_table_mark
+def test_skipfooter(dd_read, pd_read, files):
+    files = {name: content + b"\n" + comment_footer for name, content in files.items()}
+    skip = len(comment_footer.splitlines())
+    with filetexts(files, mode="b"):
+        df = dd_read("2014-01-*.csv", skipfooter=skip, engine="python")
+        expected_df = pd.concat(
+            [pd_read(n, skipfooter=skip, engine="python") for n in sorted(files)]
+        )
+        assert_eq(df, expected_df, check_dtype=False)
+
+
 @pytest.mark.parametrize(
     "dd_read,pd_read,files,units",
     [