Repository: modin-project/modin

Issue:
Row index does not match pandas on read_csv with skiprows and names on certain files
pandas has a very strange edge case here. The following conditions must be met:

* `skiprows` must be > 1
* `names` must not be `None`
* `index_col` must be `None`
* File must have different data types in each line above `skiprows` number

Since this is data dependent, the best course of action is to hand the decision of what number to start to pandas.


Discussion:
This issue has been mentioned on **Modin Discuss**. There might be relevant details there:

https://discuss.modin.org/t/modin-errors-out-on-pytz-timezone/119/33


Resolving patch:
diff --git a/modin/engines/base/io/text/csv_reader.py b/modin/engines/base/io/text/csv_reader.py
--- a/modin/engines/base/io/text/csv_reader.py
+++ b/modin/engines/base/io/text/csv_reader.py
@@ -180,11 +180,6 @@ def _read(cls, filepath_or_buffer, **kwargs):
         if index_col is None:
             row_lengths = cls.materialize(index_ids)
             new_index = pandas.RangeIndex(sum(row_lengths))
-            # pandas has a really weird edge case here.
-            if kwargs.get("names", None) is not None and skiprows > 1:
-                new_index = pandas.RangeIndex(
-                    skiprows - 1, new_index.stop + skiprows - 1
-                )
         else:
             index_objs = cls.materialize(index_ids)
             row_lengths = [len(o) for o in index_objs]


Test patch:
diff --git a/modin/pandas/test/data/issue_2239.csv b/modin/pandas/test/data/issue_2239.csv
new file mode 100644
--- /dev/null
+++ b/modin/pandas/test/data/issue_2239.csv
@@ -0,0 +1,146 @@
+1585542839.000000, 1585542839.000000, 1585542839.000000
+32.000000, 32.000000, 32.000000
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-13,51
+-38,-14,51
+-38,-14,50
+-38,-13,51
+-38,-14,50
+-38,-14,51
+-38,-13,51
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1113,7 +1113,7 @@ def test_from_csv_chunksize(make_csv_file):
     df_equals(modin_df, pd_df)
 
 
-@pytest.mark.parametrize("nrows", [123, None])
+@pytest.mark.parametrize("nrows", [1, 2, 123, None])
 def test_from_csv_skiprows(make_csv_file, nrows):
     make_csv_file()
 
@@ -1129,6 +1129,22 @@ def test_from_csv_skiprows(make_csv_file, nrows):
     )
     df_equals(modin_df, pandas_df)
 
+    pandas_df = pandas.read_csv(
+        TEST_CSV_FILENAME,
+        header=None,
+        names=["c1", "c2", "c3", "c4"],
+        skiprows=2,
+        nrows=nrows,
+    )
+    modin_df = pd.read_csv(
+        TEST_CSV_FILENAME,
+        header=None,
+        names=["c1", "c2", "c3", "c4"],
+        skiprows=2,
+        nrows=nrows,
+    )
+    df_equals(modin_df, pandas_df)
+
     pandas_df = pandas.read_csv(
         TEST_CSV_FILENAME,
         names=["c1", "c2", "c3", "c4"],
@@ -1144,6 +1160,15 @@ def test_from_csv_skiprows(make_csv_file, nrows):
     df_equals(modin_df, pandas_df)
 
 
+@pytest.mark.parametrize("names", [list("XYZ"), None])
+@pytest.mark.parametrize("skiprows", [1, 2, 3, 4, None])
+def test_from_csv_skiprows_names(names, skiprows):
+    path = "modin/pandas/test/data/issue_2239.csv"
+    pandas_df = pandas.read_csv(path, names=names, skiprows=skiprows)
+    modin_df = pd.read_csv(path, names=names, skiprows=skiprows)
+    df_equals(pandas_df, modin_df)
+
+
 @pytest.mark.parametrize(
     "encoding", ["latin8", "ISO-8859-1", "latin1", "iso-8859-1", "cp1252", "utf8"]
 )