Repository: modin-project/modin

Issue:
Update test_io.py with call to df_equals
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:

Ubuntu 19.10

- **Modin version** (`modin.__version__`):

0.7.3+2.g9f3f304

- **Python version**:

Python 3.7.5

- **Code we can use to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

Currently `modin_df_equals_pandas` is used in `test_io.py` while all other tests use `df_equals`. The problem with `modin_df_equals_pandas` doesn't test all metadata equality and was deprecated for this reason. Test `test_io.py` should be updated to match other test suites.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/text/csv_reader.py b/modin/engines/base/io/text/csv_reader.py
--- a/modin/engines/base/io/text/csv_reader.py
+++ b/modin/engines/base/io/text/csv_reader.py
@@ -66,7 +66,7 @@ def read(cls, filepath_or_buffer, **kwargs):
             # be assigned correctly
             names = pandas.read_csv(
                 filepath_or_buffer,
-                **dict(kwargs, nrows=0, skipfooter=0, index_col=None)
+                **dict(kwargs, usecols=None, nrows=0, skipfooter=0, index_col=None)
             ).columns
         empty_pd_df = pandas.read_csv(
             filepath_or_buffer, **dict(kwargs, nrows=0, skipfooter=0)


Test patch:
diff --git a/modin/experimental/pandas/test/test_io_exp.py b/modin/experimental/pandas/test/test_io_exp.py
--- a/modin/experimental/pandas/test/test_io_exp.py
+++ b/modin/experimental/pandas/test/test_io_exp.py
@@ -16,7 +16,7 @@
 import pytest
 import modin.experimental.pandas as pd
 from modin.pandas.test.test_io import (  # noqa: F401
-    modin_df_equals_pandas,
+    df_equals,
     make_sql_connection,
 )
 
@@ -40,8 +40,8 @@ def test_from_sql_distributed(make_sql_connection):  # noqa: F811
             table, conn, partition_column="col1", lower_bound=0, upper_bound=6
         )
 
-        assert modin_df_equals_pandas(modin_df_from_query, pandas_df)
-        assert modin_df_equals_pandas(modin_df_from_table, pandas_df)
+        df_equals(modin_df_from_query, pandas_df)
+        df_equals(modin_df_from_table, pandas_df)
 
 
 @pytest.mark.skipif(
@@ -60,5 +60,5 @@ def test_from_sql_defaults(make_sql_connection):  # noqa: F811
     with pytest.warns(UserWarning):
         modin_df_from_table = pd.read_sql(table, conn)
 
-    assert modin_df_equals_pandas(modin_df_from_query, pandas_df)
-    assert modin_df_equals_pandas(modin_df_from_table, pandas_df)
+    df_equals(modin_df_from_query, pandas_df)
+    df_equals(modin_df_from_table, pandas_df)
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -57,15 +57,6 @@
 SMALL_ROW_SIZE = 2000
 
 
-def modin_df_equals_pandas(modin_df, pandas_df):
-    df1 = to_pandas(modin_df).sort_index()
-    df2 = pandas_df.sort_index()
-    if os.environ.get("MODIN_BACKEND", "Pandas").lower() == "pyarrow":
-        if not df1.dtypes.equals(df2.dtypes):
-            return df2.astype(df1.dtypes).equals(df1)
-    return df1.equals(df2)
-
-
 @pytest.fixture
 def make_parquet_file():
     """Pytest fixture factory that makes a parquet file/dir for testing.
@@ -465,7 +456,7 @@ def test_from_json():
     pandas_df = pandas.read_json(TEST_JSON_FILENAME)
     modin_df = pd.read_json(TEST_JSON_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_json_file()
 
@@ -475,7 +466,7 @@ def test_from_json_lines():
 
     pandas_df = pandas.read_json(TEST_JSON_FILENAME, lines=True)
     modin_df = pd.read_json(TEST_JSON_FILENAME, lines=True)
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_json_file()
 
@@ -498,7 +489,7 @@ def test_from_html():
     pandas_df = pandas.read_html(TEST_HTML_FILENAME)[0]
     modin_df = pd.read_html(TEST_HTML_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_html_file()
 
@@ -510,7 +501,7 @@ def test_from_clipboard():
     pandas_df = pandas.read_clipboard()
     modin_df = pd.read_clipboard()
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_excel():
@@ -519,7 +510,7 @@ def test_from_excel():
     pandas_df = pandas.read_excel(TEST_EXCEL_FILENAME)
     modin_df = pd.read_excel(TEST_EXCEL_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_excel_file()
 
@@ -536,7 +527,7 @@ def test_from_excel_all_sheets():
     assert pandas_df.keys() == modin_df.keys()
 
     for key in pandas_df.keys():
-        assert modin_df_equals_pandas(modin_df.get(key), pandas_df.get(key))
+        df_equals(modin_df.get(key), pandas_df.get(key))
 
     teardown_excel_file()
 
@@ -548,7 +539,7 @@ def test_from_feather():
     pandas_df = pandas.read_feather(TEST_FEATHER_FILENAME)
     modin_df = pd.read_feather(TEST_FEATHER_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_feather_file()
 
@@ -560,7 +551,7 @@ def test_from_hdf():
     pandas_df = pandas.read_hdf(TEST_READ_HDF_FILENAME, key="df")
     modin_df = pd.read_hdf(TEST_READ_HDF_FILENAME, key="df")
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_hdf_file()
 
@@ -572,7 +563,7 @@ def test_from_hdf_format():
     pandas_df = pandas.read_hdf(TEST_READ_HDF_FILENAME, key="df")
     modin_df = pd.read_hdf(TEST_READ_HDF_FILENAME, key="df")
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_hdf_file()
 
@@ -583,7 +574,7 @@ def test_from_stata():
     pandas_df = pandas.read_stata(TEST_STATA_FILENAME)
     modin_df = pd.read_stata(TEST_STATA_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_stata_file()
 
@@ -594,7 +585,7 @@ def test_from_pickle():
     pandas_df = pandas.read_pickle(TEST_PICKLE_FILENAME)
     modin_df = pd.read_pickle(TEST_PICKLE_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     teardown_pickle_file()
 
@@ -608,12 +599,12 @@ def test_from_sql(make_sql_connection):
     pandas_df = pandas.read_sql(query, conn)
     modin_df = pd.read_sql(query, conn)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     pandas_df = pandas.read_sql(query, conn, index_col="index")
     modin_df = pd.read_sql(query, conn, index_col="index")
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     with pytest.warns(UserWarning):
         pd.read_sql_query(query, conn)
@@ -626,14 +617,14 @@ def test_from_sql(make_sql_connection):
     pandas_df = pandas.read_sql(query, conn)
     modin_df = pd.read_sql(query, conn)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     # Test SQLAlchemy Connection
     conn = conn.connect()
     pandas_df = pandas.read_sql(query, conn)
     modin_df = pd.read_sql(query, conn)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_sql_with_chunksize(make_sql_connection):
@@ -645,7 +636,7 @@ def test_from_sql_with_chunksize(make_sql_connection):
     pandas_gen = pandas.read_sql(query, conn, chunksize=10)
     modin_gen = pd.read_sql(query, conn, chunksize=10)
     for modin_df, pandas_df in zip(modin_gen, pandas_gen):
-        assert modin_df_equals_pandas(modin_df, pandas_df)
+        df_equals(modin_df, pandas_df)
 
 
 @pytest.mark.skip(reason="No SAS write methods in Pandas")
@@ -653,7 +644,7 @@ def test_from_sas():
     pandas_df = pandas.read_sas(TEST_SAS_FILENAME)
     modin_df = pd.read_sas(TEST_SAS_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv(make_csv_file):
@@ -725,7 +716,7 @@ def test_from_csv_zip(make_csv_file):
 def test_parse_dates_read_csv():
     pandas_df = pandas.read_csv("modin/pandas/test/data/test_time_parsing.csv")
     modin_df = pd.read_csv("modin/pandas/test/data/test_time_parsing.csv")
-    modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     pandas_df = pandas.read_csv(
         "modin/pandas/test/data/test_time_parsing.csv",
@@ -759,7 +750,7 @@ def test_parse_dates_read_csv():
         index_col=0,
         encoding="utf-8",
     )
-    modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     pandas_df = pandas.read_csv(
         "modin/pandas/test/data/test_time_parsing.csv",
@@ -795,7 +786,7 @@ def test_parse_dates_read_csv():
         parse_dates=["timestamp"],
         encoding="utf-8",
     )
-    modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     pandas_df = pandas.read_csv(
         "modin/pandas/test/data/test_time_parsing.csv",
@@ -831,29 +822,22 @@ def test_parse_dates_read_csv():
         parse_dates=["timestamp"],
         encoding="utf-8",
     )
-    modin_df_equals_pandas(modin_df, pandas_df)
-
-
-def test_from_csv_with_args():
-    pandas_df = pandas.read_csv(
-        "modin/pandas/test/data/issue_621.csv", header=None, usecols=[0, 7]
-    )
-    modin_df = pd.read_csv(
-        "modin/pandas/test/data/issue_621.csv", header=None, usecols=[0, 7]
-    )
-    modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
-    pandas_df = pandas.read_csv("modin/pandas/test/data/issue_621.csv", usecols=[0, 7])
-    modin_df = pd.read_csv("modin/pandas/test/data/issue_621.csv", usecols=[0, 7])
-    modin_df_equals_pandas(modin_df, pandas_df)
 
-    pandas_df = pandas.read_csv(
-        "modin/pandas/test/data/issue_621.csv", usecols=[0, 7], names=[0, 7]
-    )
-    modin_df = pd.read_csv(
-        "modin/pandas/test/data/issue_621.csv", usecols=[0, 7], names=[0, 7]
-    )
-    modin_df_equals_pandas(modin_df, pandas_df)
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"header": None, "usecols": [0, 7]},
+        {"usecols": [0, 7]},
+        {"names": [0, 7], "usecols": [0, 7]},
+    ],
+)
+def test_from_csv_with_args(kwargs):
+    file_name = "modin/pandas/test/data/issue_621.csv"
+    pandas_df = pandas.read_csv(file_name, **kwargs)
+    modin_df = pd.read_csv(file_name, **kwargs)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_table(make_csv_file):
@@ -862,12 +846,12 @@ def test_from_table(make_csv_file):
     pandas_df = pandas.read_table(TEST_CSV_FILENAME)
     modin_df = pd.read_table(TEST_CSV_FILENAME)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     pandas_df = pandas.read_table(Path(TEST_CSV_FILENAME))
     modin_df = pd.read_table(Path(TEST_CSV_FILENAME))
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 @pytest.mark.parametrize("usecols", [["a"], ["a", "b", "e"], [0, 1, 4]])
@@ -896,7 +880,7 @@ def test_from_csv_s3(make_csv_file):
         "defaulting to pandas implementation" in str(err) for err in record.list
     )
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv_default(make_csv_file):
@@ -907,7 +891,7 @@ def test_from_csv_default(make_csv_file):
     with pytest.warns(UserWarning):
         modin_df = pd.read_csv(dataset_url)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv_chunksize(make_csv_file):
@@ -919,7 +903,7 @@ def test_from_csv_chunksize(make_csv_file):
     pd_reader = pandas.read_csv(TEST_CSV_FILENAME, chunksize=500)
 
     for modin_df, pd_df in zip(rdf_reader, pd_reader):
-        assert modin_df_equals_pandas(modin_df, pd_df)
+        df_equals(modin_df, pd_df)
 
     # Tests that get_chunk works correctly
     rdf_reader = pd.read_csv(TEST_CSV_FILENAME, chunksize=1)
@@ -928,7 +912,7 @@ def test_from_csv_chunksize(make_csv_file):
     modin_df = rdf_reader.get_chunk(1)
     pd_df = pd_reader.get_chunk(1)
 
-    assert modin_df_equals_pandas(modin_df, pd_df)
+    df_equals(modin_df, pd_df)
 
     # Tests that read works correctly
     rdf_reader = pd.read_csv(TEST_CSV_FILENAME, chunksize=1)
@@ -937,7 +921,7 @@ def test_from_csv_chunksize(make_csv_file):
     modin_df = rdf_reader.read()
     pd_df = pd_reader.read()
 
-    assert modin_df_equals_pandas(modin_df, pd_df)
+    df_equals(modin_df, pd_df)
 
 
 def test_from_csv_skiprows(make_csv_file):
@@ -973,7 +957,7 @@ def test_from_csv_encoding(make_csv_file, encoding):
     pandas_df = pandas.read_csv(TEST_CSV_FILENAME, encoding=encoding)
     modin_df = pd.read_csv(TEST_CSV_FILENAME, encoding=encoding)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv_default_to_pandas_behavior(make_csv_file):
@@ -998,7 +982,7 @@ def test_from_csv_index_col(make_csv_file):
 
     pandas_df = pandas.read_csv(TEST_CSV_FILENAME, index_col="col1")
     modin_df = pd.read_csv(TEST_CSV_FILENAME, index_col="col1")
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv_skipfooter(make_csv_file):
@@ -1007,7 +991,7 @@ def test_from_csv_skipfooter(make_csv_file):
     pandas_df = pandas.read_csv(TEST_CSV_FILENAME, skipfooter=13)
     modin_df = pd.read_csv(TEST_CSV_FILENAME, skipfooter=13)
 
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv_parse_dates(make_csv_file):
@@ -1015,19 +999,19 @@ def test_from_csv_parse_dates(make_csv_file):
 
     pandas_df = pandas.read_csv(TEST_CSV_FILENAME, parse_dates=[["col2", "col4"]])
     modin_df = pd.read_csv(TEST_CSV_FILENAME, parse_dates=[["col2", "col4"]])
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
     pandas_df = pandas.read_csv(
         TEST_CSV_FILENAME, parse_dates={"time": ["col2", "col4"]}
     )
     modin_df = pd.read_csv(TEST_CSV_FILENAME, parse_dates={"time": ["col2", "col4"]})
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_csv_newlines_in_quotes():
     pandas_df = pandas.read_csv("modin/pandas/test/data/newlines.csv")
     modin_df = pd.read_csv("modin/pandas/test/data/newlines.csv")
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
 
 @pytest.mark.skip(reason="No clipboard on Travis")
@@ -1072,7 +1056,7 @@ def test_series_to_csv():
     modin_s.to_csv(TEST_CSV_DF_FILENAME)
     pandas_s.to_csv(TEST_CSV_pandas_FILENAME)
 
-    assert modin_df_equals_pandas(modin_s, pandas_s)
+    df_equals(modin_s, pandas_s)
     assert modin_s.name == pandas_s.name
     assert assert_files_eq(TEST_CSV_DF_FILENAME, TEST_CSV_pandas_FILENAME)