Repository: modin-project/modin

Issue:
Implement `to_csv` function with OmniSci backend

to_csv function defaults to pandas because it is not supported by ExperimentalHdkOnNative
Currently when calling `pd.to_csv` function to write data to a CSV file I am getting a message `UserWarning: `to_csv` is not currently supported by ExperimentalHdkOnNative, defaulting to pandas implementation.` Because of single-threaded Pandas implementation writing large datasets takes very long time.

To reproduce with real-life workload checkout this repo https://github.com/gshimansky/data-science-processing-workload and run command like this: `python launcher.py -m taxi --hdk -np` (replace taxi with census or plasticc for other two benchmarks).

I created a corresponding issue in HDK https://github.com/intel-ai/hdk/issues/468 about it because I suppose some HDK support is required to implement this feature on Modin.


Discussion:
@amyskov is this feature still unimplemented? Should we assign someone to this?
It’s still unimplemented


Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
@@ -37,7 +37,8 @@
 
 import pandas
 import pandas._libs.lib as lib
-from pandas.io.common import is_url
+from pandas.core.dtypes.common import is_list_like
+from pandas.io.common import is_url, get_handle
 
 from modin.utils import _inherit_docstrings
 
@@ -513,6 +514,109 @@ def _validate_read_csv_kwargs(
         if on_bad_lines not in ["error", "warn", "skip", None]:
             raise ValueError(f"Argument {on_bad_lines} is invalid for on_bad_lines.")
 
+    @classmethod
+    @_inherit_docstrings(BaseIO.to_csv, apilink="pandas.to_csv")
+    def to_csv(cls, qc, **kwargs):
+        df = qc._modin_frame
+        write_opts = pa.csv.WriteOptions(include_header=True, delimiter=",")
+        for key, value in kwargs.items():
+            if value is None:
+                pass
+            elif key == "sep":
+                write_opts.delimiter = value
+            elif key == "chunksize":
+                write_opts.batch_size = value
+            elif not (
+                (key == "na_rep" and len(value) == 0)
+                or (key == "decimal" and value == ".")
+                or (key == "quotechar" and value == '"')
+                or (key == "doublequote" and value is True)
+                or (key == "encoding" and value == "utf-8")
+                or (key == "lineterminator" and value == os.linesep)
+                or key
+                in (
+                    "path_or_buf",
+                    "columns",
+                    "header",
+                    "index",
+                    "index_label",
+                    "mode",
+                    "compression",
+                    "errors",
+                    "storage_options",
+                )
+            ):
+                ErrorMessage.default_to_pandas(f"Argument {key}={value}")
+                return df.to_pandas().to_csv(**kwargs)
+
+        df._execute()
+        at = df._partitions[0][0].get()
+        if not isinstance(at, pa.Table):
+            return df.to_pandas().to_csv(**kwargs)
+        idx_names = df._index_cols
+
+        if kwargs.get("index", True):
+            if idx_names is None:  # Trivial index
+                idx_col = pa.array(range(len(df.index)), type=pa.int64())
+                at = at.add_column(0, "", idx_col)
+            if (idx_names := kwargs.get("index_label", None)) is None:
+                idx_names = df.index.names
+            elif idx_names is False:
+                idx_names = [""] * len(df.index.names)
+            elif not is_list_like(idx_names):
+                idx_names = [idx_names]
+            idx_names = ["" if n is None else str(n) for n in idx_names]
+            at = at.rename_columns(idx_names + df.columns.tolist())
+        elif idx_names is not None:
+            at = at.drop(idx_names)
+            at = at.rename_columns(df.columns.tolist())
+            idx_names = None
+        else:
+            at = at.rename_columns(df.columns.tolist())
+
+        if (value := kwargs.get("columns", None)) is not None:
+            if idx_names is not None:
+                value = idx_names + value
+            at = at.select(value)
+
+        if (value := kwargs.get("header", None)) is False:
+            write_opts.include_header = False
+        elif isinstance(value, list):
+            if idx_names is not None:
+                value = idx_names + value
+            at = at.rename_columns(value)
+
+        def write_header(out):
+            # Using pandas to write the header, because pyarrow
+            # writes column names enclosed in double quotes.
+            if write_opts.include_header:
+                pdf = pandas.DataFrame(columns=at.column_names)
+                pdf.to_csv(out, sep=write_opts.delimiter, index=False)
+                write_opts.include_header = False
+
+        if (path_or_buf := kwargs.get("path_or_buf", None)) is None:
+            out = pa.BufferOutputStream()
+            write_header(out)
+            pa.csv.write_csv(at, out, write_opts)
+            return out.getvalue().to_pybytes().decode()
+
+        # Pyarrow fails to write in text mode.
+        mode = kwargs.get("mode", "w").replace("t", "")
+        if "b" not in mode:
+            mode += "b"
+
+        with get_handle(
+            path_or_buf=path_or_buf,
+            mode=mode,
+            errors=kwargs.get("errors", "strict"),
+            compression=kwargs.get("compression", "infer"),
+            storage_options=kwargs.get("storage_options", None),
+            is_text=False,
+        ) as handles:
+            out = handles.handle
+            write_header(out)
+            pa.csv.write_csv(at, out, write_opts)
+
     @classmethod
     @_inherit_docstrings(BaseIO.read_sql, apilink="pandas.read_sql")
     def read_sql(cls, **kwargs):


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -212,6 +212,44 @@ def eval_to_file(tmp_dir, modin_obj, pandas_obj, fn, extension, **fn_kwargs):
     assert assert_files_eq(unique_filename_modin, unique_filename_pandas)
 
 
+def eval_to_csv_file(tmp_dir, modin_obj, pandas_obj, extension, **kwargs):
+    if extension is None:
+        kwargs["mode"] = "t"
+        kwargs["compression"] = "infer"
+        modin_csv = modin_obj.to_csv(**kwargs)
+        pandas_csv = pandas_obj.to_csv(**kwargs)
+        if modin_csv == pandas_csv:
+            return
+
+        force_read = True
+        modin_file = get_unique_filename(extension="csv", data_dir=tmp_dir)
+        pandas_file = get_unique_filename(extension="csv", data_dir=tmp_dir)
+        with open(modin_file, "w") as file:
+            file.write(modin_csv)
+        with open(pandas_file, "w") as file:
+            file.write(pandas_csv)
+    else:
+        force_read = extension != "csv" or kwargs.get("compression", None)
+        modin_file = get_unique_filename(extension=extension, data_dir=tmp_dir)
+        pandas_file = get_unique_filename(extension=extension, data_dir=tmp_dir)
+        modin_obj.to_csv(modin_file, **kwargs)
+        pandas_obj.to_csv(pandas_file, **kwargs)
+
+    if force_read or not assert_files_eq(modin_file, pandas_file):
+        # If the files are not identical, make sure they can
+        # be read by pandas and contains identical data.
+        read_kwargs = {}
+        if kwargs.get("index", None) is not False:
+            read_kwargs["index_col"] = 0
+        if (value := kwargs.get("sep", None)) is not None:
+            read_kwargs["sep"] = value
+        if (value := kwargs.get("compression", None)) is not None:
+            read_kwargs["compression"] = value
+        modin_obj = pandas.read_csv(modin_file, **read_kwargs)
+        pandas_obj = pandas.read_csv(pandas_file, **read_kwargs)
+        df_equals(pandas_obj, modin_obj)
+
+
 @pytest.fixture
 def make_parquet_dir(tmp_path):
     def _make_parquet_dir(
@@ -1104,34 +1142,52 @@ def test_read_csv_wrong_path(self):
             filepath_or_buffer="/some/wrong/path.csv",
         )
 
-    @pytest.mark.skipif(
-        StorageFormat.get() == "Hdk",
-        reason="to_csv is not implemented with HDK storage format yet - issue #3082",
-    )
-    @pytest.mark.parametrize("header", [False, True])
+    @pytest.mark.parametrize("extension", [None, "csv", "csv.gz"])
+    @pytest.mark.parametrize("sep", [" "])
+    @pytest.mark.parametrize("header", [False, True, "sfx-"])
     @pytest.mark.parametrize("mode", ["w", "wb+"])
+    @pytest.mark.parametrize("idx_name", [None, "Index"])
+    @pytest.mark.parametrize("index", [True, False, "New index"])
+    @pytest.mark.parametrize("index_label", [None, False, "New index"])
+    @pytest.mark.parametrize("columns", [None, ["col1", "col3", "col5"]])
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",
         reason="The reason of tests fail in `cloud` mode is unknown for now - issue #2340",
     )
-    def test_to_csv(self, tmp_path, header, mode):
-        pandas_df = generate_dataframe()
+    def test_to_csv(
+        self,
+        tmp_path,
+        extension,
+        sep,
+        header,
+        mode,
+        idx_name,
+        index,
+        index_label,
+        columns,
+    ):
+        pandas_df = generate_dataframe(idx_name=idx_name)
         modin_df = pd.DataFrame(pandas_df)
 
-        eval_to_file(
+        if isinstance(header, str):
+            if columns is None:
+                header = [f"{header}{c}" for c in modin_df.columns]
+            else:
+                header = [f"{header}{c}" for c in columns]
+
+        eval_to_csv_file(
             tmp_path,
             modin_obj=modin_df,
             pandas_obj=pandas_df,
-            fn="to_csv",
-            extension="csv",
+            extension=extension,
+            sep=sep,
             header=header,
             mode=mode,
+            index=index,
+            index_label=index_label,
+            columns=columns,
         )
 
-    @pytest.mark.skipif(
-        StorageFormat.get() == "Hdk",
-        reason="to_csv is not implemented with HDK storage format yet - issue #3082",
-    )
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",
         reason="The reason of tests fail in `cloud` mode is unknown for now - issue #2340",
@@ -1139,18 +1195,13 @@ def test_to_csv(self, tmp_path, header, mode):
     def test_dataframe_to_csv(self, tmp_path):
         pandas_df = pandas.read_csv(pytest.csvs_names["test_read_csv_regular"])
         modin_df = pd.DataFrame(pandas_df)
-        eval_to_file(
+        eval_to_csv_file(
             tmp_path,
             modin_obj=modin_df,
             pandas_obj=pandas_df,
-            fn="to_csv",
             extension="csv",
         )
 
-    @pytest.mark.skipif(
-        StorageFormat.get() == "Hdk",
-        reason="to_csv is not implemented with HDK storage format yet - issue #3082",
-    )
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",
         reason="The reason of tests fail in `cloud` mode is unknown for now - issue #2340",
@@ -1160,11 +1211,10 @@ def test_series_to_csv(self, tmp_path):
             pytest.csvs_names["test_read_csv_regular"], usecols=["col1"]
         ).squeeze()
         modin_s = pd.Series(pandas_s)
-        eval_to_file(
+        eval_to_csv_file(
             tmp_path,
             modin_obj=modin_s,
             pandas_obj=pandas_s,
-            fn="to_csv",
             extension="csv",
         )
 
@@ -1297,7 +1347,7 @@ def test_to_csv_with_index(self, tmp_path):
             values,
             columns=["key"] + ["avalue" + str(i) for i in range(1, 1 + cols)],
         ).set_index("key")
-        eval_to_file(tmp_path, modin_df, pandas_df, "to_csv", "csv")
+        eval_to_csv_file(tmp_path, modin_df, pandas_df, "csv")
 
     @pytest.mark.parametrize("set_async_read_mode", [False, True], indirect=True)
     def test_read_csv_issue_5150(self, set_async_read_mode):
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -1335,7 +1335,7 @@ def wrapped_function(self, *args, **kwargs):
     return wrapper
 
 
-def generate_dataframe(row_size=NROWS, additional_col_values=None):
+def generate_dataframe(row_size=NROWS, additional_col_values=None, idx_name=None):
     dates = pandas.date_range("2000", freq="h", periods=row_size)
     data = {
         "col1": np.arange(row_size) * 10,
@@ -1345,11 +1345,12 @@ def generate_dataframe(row_size=NROWS, additional_col_values=None):
         "col5": [get_random_string() for _ in range(row_size)],
         "col6": random_state.uniform(low=0.0, high=10000.0, size=row_size),
     }
+    index = None if idx_name is None else pd.RangeIndex(0, row_size, name=idx_name)
 
     if additional_col_values is not None:
         assert isinstance(additional_col_values, (list, tuple))
         data.update({"col7": random_state.choice(additional_col_values, size=row_size)})
-    return pandas.DataFrame(data)
+    return pandas.DataFrame(data, index=index)
 
 
 def _make_csv_file(filenames):