internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
PERF: remove df.copy call from `from_pandas` as it seems unnecessary for Ray and Dask backends
Place: https://github.com/modin-project/modin/blob/a28e319d9756c9383afa9efd67e32eb04e6b848c/modin/core/dataframe/pandas/partitioning/partition_manager.py#L745

Note: there was a https://github.com/modin-project/modin/pull/276 that add this copy for fixing python backend. I think it's possible to move the copy call into `put_func` for python backend.


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/partitioning/partition_manager.py b/modin/core/dataframe/pandas/partitioning/partition_manager.py
--- a/modin/core/dataframe/pandas/partitioning/partition_manager.py
+++ b/modin/core/dataframe/pandas/partitioning/partition_manager.py
@@ -741,9 +741,7 @@ def update_bar(pbar, f):
             [
                 update_bar(
                     pbar,
-                    put_func(
-                        df.iloc[i : i + row_chunksize, j : j + col_chunksize].copy()
-                    ),
+                    put_func(df.iloc[i : i + row_chunksize, j : j + col_chunksize]),
                 )
                 for j in range(0, len(df.columns), col_chunksize)
             ]
diff --git a/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py
--- a/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py
+++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py
@@ -166,7 +166,7 @@ def put(cls, obj):
         PandasOnPythonDataframePartition
             New ``PandasOnPythonDataframePartition`` object.
         """
-        return cls(obj)
+        return cls(obj.copy())
 
     @classmethod
     def preprocess_func(cls, func):
diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py
--- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py
+++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py
@@ -112,7 +112,7 @@ def put(cls, obj):
             The new partition.
         """
         return OmnisciOnNativeDataframePartition(
-            pandas_df=obj, length=len(obj.index), width=len(obj.columns)
+            pandas_df=obj.copy(), length=len(obj.index), width=len(obj.columns)
         )
 
     def wait(self):


Test patch:
Repository: modin-project/modin

Issue:
REFACTOR: unify `split_result_of_axis_func_pandas`



Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/utils.py b/modin/core/storage_formats/pandas/utils.py
--- a/modin/core/storage_formats/pandas/utils.py
+++ b/modin/core/storage_formats/pandas/utils.py
@@ -92,23 +92,16 @@ def split_result_of_axis_func_pandas(axis, num_splits, result, length_list=None)
 
     if length_list is not None:
         length_list.insert(0, 0)
-        sums = np.cumsum(length_list)
-        if axis == 0 or isinstance(result, pandas.Series):
-            return [result.iloc[sums[i] : sums[i + 1]] for i in range(len(sums) - 1)]
-        else:
-            return [result.iloc[:, sums[i] : sums[i + 1]] for i in range(len(sums) - 1)]
-
+    else:
+        chunksize = compute_chunksize(result.shape[axis], num_splits)
+        length_list = np.full(num_splits + 1, chunksize)
+        length_list[0] = 0
+    sums = np.cumsum(length_list)
     # We do this to restore block partitioning
-    chunksize = compute_chunksize(result.shape[axis], num_splits)
     if axis == 0 or isinstance(result, pandas.Series):
-        return [
-            result.iloc[chunksize * i : chunksize * (i + 1)] for i in range(num_splits)
-        ]
+        return [result.iloc[sums[i] : sums[i + 1]] for i in range(len(sums) - 1)]
     else:
-        return [
-            result.iloc[:, chunksize * i : chunksize * (i + 1)]
-            for i in range(num_splits)
-        ]
+        return [result.iloc[:, sums[i] : sums[i + 1]] for i in range(len(sums) - 1)]
 
 
 def length_fn_pandas(df):


Test patch:
Repository: modin-project/modin

Issue:
PERF: `copy` should not trigger any previous computations 
Place: https://github.com/modin-project/modin/blob/a07ec9d539453fea8e682627702ad05675fad805/modin/core/dataframe/pandas/dataframe/dataframe.py#L1005


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -1002,10 +1002,10 @@ def copy(self):
         """
         return self.__constructor__(
             self._partitions,
-            self.index.copy(),
-            self.columns.copy(),
-            self._row_lengths,
-            self._column_widths,
+            self._index_cache.copy() if self._index_cache is not None else None,
+            self._columns_cache.copy() if self._columns_cache is not None else None,
+            self._row_lengths_cache,
+            self._column_widths_cache,
             self._dtypes,
         )
 


Test patch:
Repository: modin-project/modin

Issue:
PERF: compute dtypes in `concat` also for ROW_WISE case when possible



Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -2649,11 +2649,19 @@ def _compute_new_widths():
         new_partitions = self._partition_mgr_cls.concat(
             axis.value, left_parts, right_parts
         )
+        new_dtypes = None
         if axis == Axis.ROW_WISE:
             new_index = self.index.append([other.index for other in others])
             new_columns = joined_index
-            # TODO: Can optimize by combining if all dtypes are materialized
-            new_dtypes = None
+            all_dtypes = [frame._dtypes for frame in [self] + others]
+            if all(dtypes is not None for dtypes in all_dtypes):
+                new_dtypes = pandas.concat(all_dtypes, axis=1)
+                # 'nan' value will be placed in a row if a column doesn't exist in all frames;
+                # this value is np.float64 type so we need an explicit conversion
+                new_dtypes.fillna(np.dtype("float64"), inplace=True)
+                new_dtypes = new_dtypes.apply(
+                    lambda row: find_common_type(row.values), axis=1
+                )
             # If we have already cached the length of each row in at least one
             # of the row's partitions, we can build new_lengths for the new
             # frame. Typically, if we know the length for any partition in a
@@ -2672,8 +2680,6 @@ def _compute_new_widths():
             new_index = joined_index
             if self._dtypes is not None and all(o._dtypes is not None for o in others):
                 new_dtypes = self.dtypes.append([o.dtypes for o in others])
-            else:
-                new_dtypes = None
             # If we have already cached the width of each column in at least one
             # of the column's partitions, we can build new_widths for the new
             # frame. Typically, if we know the width for any partition in a


Test patch:
Repository: modin-project/modin

Issue:
PERF: `PandasDataframeAxisPartition.deploy_axis_func` should be serialized only once
Place: https://github.com/modin-project/modin/blob/cfcdf081e197d35e1fd1526f5585e361793dc2fa/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/virtual_partition.py#L191

Currently, this function is serialized on every call.


Discussion:

Resolving patch:
diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/virtual_partition.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/virtual_partition.py
--- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/virtual_partition.py
+++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/virtual_partition.py
@@ -24,6 +24,9 @@
 from .partition import PandasOnRayDataframePartition
 
 
+_DEPLOY_AXIS_FUNC = ray.put(PandasDataframeAxisPartition.deploy_axis_func)
+
+
 class PandasOnRayDataframeVirtualPartition(PandasDataframeAxisPartition):
     """
     The class implements the interface in ``PandasDataframeAxisPartition``.
@@ -188,7 +191,7 @@ def deploy_axis_func(
             num_returns=(num_splits if lengths is None else len(lengths)) * 4,
             **({"max_retries": max_retries} if max_retries is not None else {}),
         ).remote(
-            PandasDataframeAxisPartition.deploy_axis_func,
+            _DEPLOY_AXIS_FUNC,
             axis,
             func,
             num_splits,


Test patch:
Repository: modin-project/modin

Issue:
Unable to create DataFrame form a PyArrow table containing a Dictionary Array
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 22.04.1
- **Modin version** (`modin.__version__`): 0.15.0+89.g8f3f3899
- **Python version**: 3.8.13
- **Code we can use to reproduce**:

import pyarrow as pa
import modin.config as cfg
from modin.pandas.utils import from_arrow
from modin.pandas.test.utils import df_equals

cfg.StorageFormat.put('omnisci')
cfg.IsExperimental.put(True)

indices = pa.array([0, 1, 0, 1, 2, 0, None, 2])
dictionary = pa.array(['first', 'second', 'third'])
dict_array = pa.DictionaryArray.from_arrays(indices, dictionary)
at = pa.table({"col": dict_array})
pdf = at.to_pandas()
print(pdf)
mdf = from_arrow(at)
print(mdf)
df_equals(mdf, pdf)

### Describe the problem
The method from_arrow() fails with the bellow error if the input table contains a dictionary. 

### Source code / logs
Traceback (most recent call last):
...
  File "pyarrow/types.pxi", line 200, in pyarrow.lib.DataType.to_pandas_dtype
NotImplementedError: dictionary<values=string, indices=int64, ordered=0>



Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -3095,6 +3095,8 @@ def _arrow_type_to_dtype(cls, arrow_type):
         except NotImplementedError:
             if pyarrow.types.is_time(arrow_type):
                 res = np.dtype(datetime.time)
+            elif pyarrow.types.is_dictionary(arrow_type):
+                res = pandas.CategoricalDtype
             else:
                 raise
 


Test patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
@@ -2366,5 +2366,16 @@ def set_axis(df, **kwargs):
         )
 
 
+class TestFromArrow:
+    def test_dict(self):
+        indices = pyarrow.array([0, 1, 0, 1, 2, 0, None, 2])
+        dictionary = pyarrow.array(["first", "second", "third"])
+        dict_array = pyarrow.DictionaryArray.from_arrays(indices, dictionary)
+        at = pyarrow.table({"col": dict_array})
+        pdf = at.to_pandas()
+        mdf = from_arrow(at)
+        df_equals(mdf, pdf)
+
+
 if __name__ == "__main__":
     pytest.main(["-v", __file__])
Repository: modin-project/modin

Issue:
PERF: `dataframe.filter` can use lazy index and columns evaluation
Place: https://github.com/modin-project/modin/blob/fbd1e2a1b91170a3c45ce9565b1051bb2d55e4eb/modin/core/dataframe/pandas/dataframe/dataframe.py#L1799

We can avoid following call: https://github.com/modin-project/modin/blob/fbd1e2a1b91170a3c45ce9565b1051bb2d55e4eb/modin/core/dataframe/pandas/dataframe/dataframe.py#L1833 


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -1827,10 +1827,7 @@ def filter(self, axis: Union[Axis, int], condition: Callable) -> "PandasDatafram
 
         new_axes[axis.value] = self.axes[axis.value]
         new_lengths[axis.value] = self._axes_lengths[axis.value]
-        (
-            new_axes[axis.value ^ 1],
-            new_lengths[axis.value ^ 1],
-        ) = self._compute_axis_labels_and_lengths(axis.value ^ 1, new_partitions)
+        new_axes[axis.value ^ 1], new_lengths[axis.value ^ 1] = None, None
 
         return self.__constructor__(
             new_partitions,


Test patch:
Repository: modin-project/modin

Issue:
PERF: `iloc` function that used in `partition.mask` should be serialized only once
Place: https://github.com/modin-project/modin/blob/cfcdf081e197d35e1fd1526f5585e361793dc2fa/modin/core/dataframe/pandas/partitioning/partition.py#L188

Motivation: `partition.mask` - one of the most used functions.


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/partitioning/partition.py b/modin/core/dataframe/pandas/partitioning/partition.py
--- a/modin/core/dataframe/pandas/partitioning/partition.py
+++ b/modin/core/dataframe/pandas/partitioning/partition.py
@@ -162,6 +162,11 @@ def to_numpy(self, **kwargs):
         """
         return self.apply(lambda df, **kwargs: df.to_numpy(**kwargs)).get()
 
+    @staticmethod
+    def _iloc(df, row_labels, col_labels):  # noqa: RT01, PR01
+        """Perform `iloc` on dataframes wrapped in partitions (helper function)."""
+        return df.iloc[row_labels, col_labels]
+
     def mask(self, row_labels, col_labels):
         """
         Lazily create a mask that extracts the indices provided.
@@ -200,7 +205,7 @@ def is_full_axis_mask(index, axis_length):
         ):
             return copy(self)
 
-        new_obj = self.add_to_apply_calls(lambda df: df.iloc[row_labels, col_labels])
+        new_obj = self.add_to_apply_calls(self._iloc, row_labels, col_labels)
 
         def try_recompute_cache(indices, previous_cache):
             """Compute new axis-length cache for the masked frame based on its previous cache."""
diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py
--- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py
+++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py
@@ -206,6 +206,8 @@ def __copy__(self):
             call_queue=self.call_queue,
         )
 
+    _iloc = ray.put(PandasDataframePartition._iloc)
+
     def mask(self, row_labels, col_labels):
         """
         Lazily create a mask that extracts the indices provided.


Test patch:
Repository: modin-project/modin

Issue:
PERF: try to avoid index materialization in `BasePandasDataset.__getattribute__`
Place: https://github.com/modin-project/modin/blob/cfcdf081e197d35e1fd1526f5585e361793dc2fa/modin/pandas/base.py#L3415

For optimization purpose we use `self.empty` check that trigger index materialization. It is most likely possible to do this check only when the index is already materialized. To do this, we can check the corresponding cache.


Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -255,6 +255,21 @@ def default_to_pandas(self, pandas_op, *args, **kwargs):
         else:
             return result
 
+    @property
+    def lazy_execution(self):
+        """
+        Whether underlying Modin frame should be executed in a lazy mode.
+
+        If True, such QueryCompiler will be handled differently at the front-end in order
+        to reduce triggering the computation as much as possible.
+
+        Returns
+        -------
+        bool
+        """
+        frame = self._modin_frame
+        return frame._index_cache is None or frame._columns_cache is None
+
     def finalize(self):
         self._modin_frame.finalize()
 


Test patch:
Repository: modin-project/modin

Issue:
BUG: `base_lengths` should be computed from `base_frame` instead of `self` in `copartition`
In some cases, `self` object may be empty, which may lead to incorrect work, for example, for `insert` function.

Place: https://github.com/modin-project/modin/blob/eee5f435f68786778184c4886ff25d386fce0c4f/modin/core/dataframe/pandas/dataframe/dataframe.py#L2486




Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -2483,7 +2483,9 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
                 base_lengths = [obj.length() for obj in reindexed_base.T[0]]
         else:
             reindexed_base = base_frame._partitions
-            base_lengths = self._column_widths if axis else self._row_lengths
+            base_lengths = (
+                base_frame._column_widths if axis else base_frame._row_lengths
+            )
 
         others_lengths = [o._axes_lengths[axis] for o in other_frames]
 


Test patch:
Repository: modin-project/modin

Issue:
PERF: avoid index and cache computations in `take_2d_labels_or_positional` unless they are needed
Places: 
https://github.com/modin-project/modin/blob/84f638cc703beb550edecce715532c8e86fb7b25/modin/core/dataframe/pandas/dataframe/dataframe.py#L724-#L728

https://github.com/modin-project/modin/blob/84f638cc703beb550edecce715532c8e86fb7b25/modin/core/dataframe/pandas/dataframe/dataframe.py#L778-#L782


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -721,11 +721,9 @@ def take_2d_labels_or_positional(
                 else sorted_row_positions
             ]
         else:
-            row_partitions_list = {
-                i: slice(None) for i in range(len(self._row_lengths))
-            }
-            new_row_lengths = self._row_lengths
-            new_index = self.index
+            row_partitions_list = {i: slice(None) for i in range(len(self._partitions))}
+            new_row_lengths = self._row_lengths_cache
+            new_index = self._index_cache
 
         # Get numpy array of positions of values from `col_labels`
         if col_labels is not None:
@@ -776,14 +774,11 @@ def take_2d_labels_or_positional(
                 new_dtypes = None
         else:
             col_partitions_list = {
-                i: slice(None) for i in range(len(self._column_widths))
+                i: slice(None) for i in range(len(self._partitions.T))
             }
-            new_col_widths = self._column_widths
-            new_columns = self.columns
-            if self._dtypes is not None:
-                new_dtypes = self.dtypes
-            else:
-                new_dtypes = None
+            new_col_widths = self._column_widths_cache
+            new_columns = self._columns_cache
+            new_dtypes = self._dtypes
         new_partitions = np.array(
             [
                 [


Test patch:
Repository: modin-project/modin

Issue:
Upgrade pandas to 1.4.4



Discussion:

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -29,7 +29,7 @@
             f"Starting Modin in compatibility mode to support legacy pandas version {__pandas_version__}"
         )
 elif PandasCompatVersion.CURRENT == PandasCompatVersion.LATEST:
-    __pandas_version__ = "1.4.3"
+    __pandas_version__ = "1.4.4"
 
     if pandas.__version__ != __pandas_version__:
         warnings.warn(
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 import versioneer
 import sys
 
-PANDAS_VERSION = "1.4.3" if sys.version_info >= (3, 8) else "1.1.5"
+PANDAS_VERSION = "1.4.4" if sys.version_info >= (3, 8) else "1.1.5"
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()


Test patch:
Repository: modin-project/modin

Issue:
BUG: `dtypes` wrong computation in `dataframe.filter`
Place: https://github.com/modin-project/modin/blob/19ac8ff208eb02a7f86037e338ce306f1e96d7a9/modin/core/dataframe/pandas/dataframe/dataframe.py#L1882


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -1970,7 +1970,7 @@ def filter(self, axis: Union[Axis, int], condition: Callable) -> "PandasDatafram
             new_partitions,
             *new_axes,
             *new_lengths,
-            self.dtypes if axis == 0 else None,
+            self.dtypes if axis == Axis.COL_WISE else None,
         )
 
     def filter_by_types(self, types: List[Hashable]) -> "PandasDataframe":


Test patch:
diff --git a/modin/pandas/test/dataframe/test_udf.py b/modin/pandas/test/dataframe/test_udf.py
--- a/modin/pandas/test/dataframe/test_udf.py
+++ b/modin/pandas/test/dataframe/test_udf.py
@@ -43,6 +43,7 @@
 )
 from modin.config import NPartitions, StorageFormat
 from modin.test.test_utils import warns_that_defaulting_to_pandas
+from modin.utils import get_current_execution
 
 NPartitions.put(4)
 
@@ -359,6 +360,10 @@ def f(x, arg2=0, arg3=0):
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 @pytest.mark.parametrize("funcs", query_func_values, ids=query_func_keys)
 def test_query(data, funcs):
+    if get_current_execution() == "BaseOnPython" and funcs != "col3 > col4":
+        pytest.xfail(
+            reason="In this case, we are faced with the problem of handling empty data frames - #4934"
+        )
     modin_df = pd.DataFrame(data)
     pandas_df = pandas.DataFrame(data)
 
@@ -369,7 +374,10 @@ def test_query(data, funcs):
             modin_df.query(funcs)
     else:
         modin_result = modin_df.query(funcs)
+        # `dtypes` must be evaluated after `query` so we need to check cache
+        assert modin_result._query_compiler._modin_frame._dtypes is not None
         df_equals(modin_result, pandas_result)
+        df_equals(modin_result.dtypes, pandas_result.dtypes)
 
 
 def test_empty_query():