Repository: modin-project/modin

Issue:
Dataframe.append works slow on omnisci engine with default arguments
with `ignore_index=True` `append` bypasses omnisci execution and concatenates arrow tables, but it is not set by default, so we need to optimize the default case as well.

<details>
<summary>Reproducer</summary>

```python
import os
import modin.config
modin.config.IsExperimental.put(True)
modin.config.StorageFormat.put("omnisci")

import pandas as pd
import modin.pandas as mpd
import numpy as np

from timeit import default_timer as timer
def generate_data(nrows, ncols):
    data = {
        f"col{i}": np.random.randint(0, 1000000000, size=nrows)
        for i in range(ncols)
    }
    return data

NROWS = 10_000_000
NCOLS = 10
df = pd.DataFrame(generate_data(NROWS, NCOLS))
df2 = pd.DataFrame(generate_data(NROWS, NCOLS))

mdf = mpd.DataFrame(generate_data(NROWS, NCOLS))
mdf2 = mpd.DataFrame(generate_data(NROWS, NCOLS))

_ = mdf.shape 
_ = mdf2.shape

print("Start pandas")
for i in range(10):
    elapsed_time = timer()
    copy = df
    copy = copy.append(df2, ignore_index=True)
    
    elapsed_time = timer() - elapsed_time
    print("Pandas time:", elapsed_time)
print("Start modin")

for i in range(10):
    elapsed_time = timer()
    copy = mdf
    copy = copy.append(mdf2)
    
    copy = copy.shape
    
    elapsed_time = timer()- elapsed_time
    print("Modin time:", elapsed_time)
```

</details>


Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py
--- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py
@@ -1689,9 +1689,7 @@ def _can_execute_arrow(self):
             )
         elif isinstance(self._op, TransformNode):
             return (
-                not self._uses_rowid
-                and self._op.is_simple_select()
-                and self._op.input[0]._can_execute_arrow()
+                self._op.is_simple_select() and self._op.input[0]._can_execute_arrow()
             )
         elif isinstance(self._op, UnionNode):
             return all(frame._can_execute_arrow() for frame in self._op.input)
@@ -1737,13 +1735,16 @@ def _arrow_select(self, exprs):
             The resulting table.
         """
         table = self._execute_arrow()
-        schema = table.schema
 
         new_fields = []
         new_columns = []
 
         for col, expr in exprs.items():
-            field = schema.field(f"F_{expr.column}")
+            if expr.column == "__rowid__" and "F___rowid__" not in table.schema.names:
+                arr = pyarrow.array(np.arange(0, table.num_rows))
+                table = table.append_column("F___rowid__", arr)
+
+            field = table.schema.field(f"F_{expr.column}")
             if col != expr.column:
                 field = field.with_name(f"F_{col}")
             new_fields.append(field)


Test patch:
diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
--- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
@@ -2026,16 +2026,9 @@ def test_shape_hint_detection_from_arrow(self):
 
 
 class TestArrowExecution:
-    data1 = {
-        "a": [1, 2, 3],
-        "b": [3, 4, 5],
-        "c": [6, 7, 8],
-    }
-    data2 = {
-        "a": [1, 2, 3],
-        "d": [3, 4, 5],
-        "e": [6, 7, 8],
-    }
+    data1 = {"a": [1, 2, 3], "b": [3, 4, 5], "c": [6, 7, 8]}
+    data2 = {"a": [1, 2, 3], "d": [3, 4, 5], "e": [6, 7, 8]}
+    data3 = {"a": [4, 5, 6], "b": [6, 7, 8], "c": [9, 10, 11]}
 
     def test_drop_rename_concat(self):
         def drop_rename_concat(df1, df2, lib, **kwargs):
@@ -2058,6 +2051,15 @@ def apply(df, **kwargs):
 
         run_and_compare(apply, data={}, force_arrow_execute=True)
 
+    def test_append(self):
+        def apply(df1, df2, **kwargs):
+            tmp = df1.append(df2)
+            return tmp
+
+        run_and_compare(
+            apply, data=self.data1, data2=self.data3, force_arrow_execute=True
+        )
+
 
 if __name__ == "__main__":
     pytest.main(["-v", __file__])