internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
Fix code scanning alert - `__init__` method calls overridden method
<!-- Warning: The suggested title contains the alert rule name. This can expose security information. -->

Tracking issue for:
- [ ] https://github.com/modin-project/modin/security/code-scanning/2



Discussion:

Resolving patch:
diff --git a/modin/pandas/accessor.py b/modin/pandas/accessor.py
--- a/modin/pandas/accessor.py
+++ b/modin/pandas/accessor.py
@@ -46,7 +46,8 @@ def __init__(self, data=None):
         self._parent = data
         self._validate(data)
 
-    def _validate(self, data):
+    @classmethod
+    def _validate(cls, data):
         """
         Verify that `data` dtypes are compatible with `pandas.core.arrays.sparse.dtype.SparseDtype`.
 
@@ -87,7 +88,8 @@ def _default_to_pandas(self, op, *args, **kwargs):
 
 @_inherit_docstrings(pandas.core.arrays.sparse.accessor.SparseFrameAccessor)
 class SparseFrameAccessor(BaseSparseAccessor):
-    def _validate(self, data):
+    @classmethod
+    def _validate(cls, data):
         """
         Verify that `data` dtypes are compatible with `pandas.core.arrays.sparse.dtype.SparseDtype`.
 
@@ -103,7 +105,7 @@ def _validate(self, data):
         """
         dtypes = data.dtypes
         if not all(isinstance(t, SparseDtype) for t in dtypes):
-            raise AttributeError(self._validation_msg)
+            raise AttributeError(cls._validation_msg)
 
     @property
     def density(self):
@@ -125,7 +127,8 @@ def to_coo(self):
 
 @_inherit_docstrings(pandas.core.arrays.sparse.accessor.SparseAccessor)
 class SparseAccessor(BaseSparseAccessor):
-    def _validate(self, data):
+    @classmethod
+    def _validate(cls, data):
         """
         Verify that `data` dtype is compatible with `pandas.core.arrays.sparse.dtype.SparseDtype`.
 
@@ -140,7 +143,7 @@ def _validate(self, data):
             If check fails.
         """
         if not isinstance(data.dtype, SparseDtype):
-            raise AttributeError(self._validation_msg)
+            raise AttributeError(cls._validation_msg)
 
     @property
     def density(self):


Test patch:
Repository: modin-project/modin

Issue:
Fix code scanning alert - Unused global variable
<!-- Warning: The suggested title contains the alert rule name. This can expose security information. -->

Tracking issue for:
- [ ] https://github.com/modin-project/modin/security/code-scanning/427



Discussion:

Resolving patch:
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -14,8 +14,9 @@
 """Module houses `ParquetDispatcher` class, that is used for reading `.parquet` files."""
 
 import os
-
+import re
 import json
+
 import fsspec
 from fsspec.core import url_to_fs
 from fsspec.spec import AbstractBufferedFile
@@ -292,6 +293,8 @@ def _get_fastparquet_files(self):  # noqa: GL08
 class ParquetDispatcher(ColumnStoreDispatcher):
     """Class handles utils for reading `.parquet` files."""
 
+    index_regex = re.compile(r"__index_level_\d+__")
+
     @classmethod
     def get_dataset(cls, path, engine, storage_options):
         """
@@ -603,8 +606,6 @@ def _read(cls, path, engine, columns, **kwargs):
         ParquetFile API is used. Please refer to the documentation here
         https://arrow.apache.org/docs/python/parquet.html
         """
-        from modin.pandas.io import PQ_INDEX_REGEX
-
         if isinstance(path, str):
             if os.path.isdir(path):
                 path_generator = os.walk(path)
@@ -650,7 +651,7 @@ def _read(cls, path, engine, columns, **kwargs):
         columns = [
             c
             for c in column_names
-            if c not in index_columns and not PQ_INDEX_REGEX.match(c)
+            if c not in index_columns and not cls.index_regex.match(c)
         ]
 
         return cls.build_query_compiler(dataset, columns, index_columns, **kwargs)
diff --git a/modin/pandas/io.py b/modin/pandas/io.py
--- a/modin/pandas/io.py
+++ b/modin/pandas/io.py
@@ -44,7 +44,6 @@
 )
 import pathlib
 import pickle
-import re
 from typing import (
     Union,
     IO,
@@ -69,8 +68,6 @@
 from modin.utils import _inherit_docstrings, Engine
 from . import _update_engine
 
-PQ_INDEX_REGEX = re.compile(r"__index_level_\d+__")
-
 
 def _read(**kwargs):
     """


Test patch:
Repository: modin-project/modin

Issue:
Fix code scanning alert - Unnecessary lambda
<!-- Warning: The suggested title contains the alert rule name. This can expose security information. -->

Tracking issue for:
- [ ] https://github.com/modin-project/modin/security/code-scanning/49



Discussion:

Resolving patch:
diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py
--- a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py
+++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py
@@ -266,7 +266,7 @@ def length(self):
             The length of the object.
         """
         if self._length_cache is None:
-            self._length_cache = self.apply(lambda df: len(df))._data
+            self._length_cache = self.apply(len)._data
         if isinstance(self._length_cache, Future):
             self._length_cache = DaskWrapper.materialize(self._length_cache)
         return self._length_cache


Test patch:
diff --git a/modin/experimental/xgboost/test/test_xgboost.py b/modin/experimental/xgboost/test/test_xgboost.py
--- a/modin/experimental/xgboost/test/test_xgboost.py
+++ b/modin/experimental/xgboost/test/test_xgboost.py
@@ -104,8 +104,8 @@ def test_xgb_with_binary_classification_datasets(data, num_actors, modin_type_y)
     predictions = bst.predict(xgb_dmatrix)
     modin_predictions = modin_bst.predict(mxgb_dmatrix)
 
-    preds = pd.DataFrame(predictions).apply(lambda x: round(x))
-    modin_preds = modin_predictions.apply(lambda x: round(x))
+    preds = pd.DataFrame(predictions).apply(round)
+    modin_preds = modin_predictions.apply(round)
 
     val = accuracy_score(y, preds)
     modin_val = accuracy_score(modin_y, modin_preds)
diff --git a/modin/pandas/test/dataframe/test_map_metadata.py b/modin/pandas/test/dataframe/test_map_metadata.py
--- a/modin/pandas/test/dataframe/test_map_metadata.py
+++ b/modin/pandas/test/dataframe/test_map_metadata.py
@@ -1399,7 +1399,7 @@ def test___hash__():
     data = test_data_values[0]
     pandas_df = pandas.DataFrame(data)
     modin_df = pd.DataFrame(data)
-    eval_general(modin_df, pandas_df, lambda df: hash(df))
+    eval_general(modin_df, pandas_df, hash)
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -280,7 +280,7 @@
 # Test functions for applymap
 test_func = {
     "plus one": lambda x: x + 1,
-    "convert to string": lambda x: str(x),
+    "convert to string": str,
     "square": lambda x: x * x,
     "identity": lambda x: x,
     "return false": lambda x: False,
Repository: modin-project/modin

Issue:
unidist on MPI fails to serialize a class derived from `NamedTuple`
Originally found in https://github.com/modin-project/modin/pull/5059#discussion_r983666941. If we revert `ParquetFileToRead` back to be derived from `NamedTuple`, tests for parquet fail on unidist on MPI. We should investigate and fix the issue.

The failed job is https://github.com/modin-project/modin/actions/runs/3524150780/jobs/5909761234.


Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/parsers.py b/modin/core/storage_formats/pandas/parsers.py
--- a/modin/core/storage_formats/pandas/parsers.py
+++ b/modin/core/storage_formats/pandas/parsers.py
@@ -48,7 +48,7 @@
 from pandas.core.dtypes.concat import union_categoricals
 from pandas.io.common import infer_compression
 from pandas.util._decorators import doc
-from typing import Any
+from typing import Any, NamedTuple
 import warnings
 
 from modin.core.io.file_dispatcher import OpenFile
@@ -642,7 +642,7 @@ def parse(fname, **kwargs):
         ]
 
 
-class ParquetFileToRead:
+class ParquetFileToRead(NamedTuple):
     """
     Class to store path and row group information for parquet reads.
 
@@ -656,10 +656,9 @@ class ParquetFileToRead:
         Row group to stop read.
     """
 
-    def __init__(self, path: Any, row_group_start: int, row_group_end: int):
-        self.path: Any = path
-        self.row_group_start: int = row_group_start
-        self.row_group_end: int = row_group_end
+    path: Any
+    row_group_start: int
+    row_group_end: int
 
 
 @doc(_doc_pandas_parser_class, data_type="PARQUET data")


Test patch:
Repository: modin-project/modin

Issue:
BUG: `_compute_axis_labels_and_lengths` works wrong with MultiIndex in some cases
https://github.com/modin-project/modin/pull/4601 found the following problem:
```bash
If we have a MultiIndex, but the first partition is empty, which may happen when
the dataframe is small, `_compute_axis_labels_and_lengths` will return us a flattened
MultiIndex - i.e. an Index consisting of tuples. This is because the MultiIndex from
the remaining partitions is appended to an empty flat Index, which results in a
flattened index. To work around this, we need to convert this flattened Index back
into a MultiIndex.
```


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -2058,18 +2058,7 @@ def sort_function(df):
                 new_axes[axis.value],
                 new_lengths[axis.value],
             ) = self._compute_axis_labels_and_lengths(axis.value, new_partitions)
-        # If we have a MultiIndex, but the first partition is empty, which may happen when
-        # the dataframe is small, `_compute_axis_labels_and_lengths` will return us a flattened
-        # MultiIndex - i.e. an Index consisting of tuples. This is because the MultiIndex from
-        # the remaining partitions is appended to an empty flat Index, which results in a
-        # flattened index. To work around this, we need to convert this flattened Index back
-        # into a MultiIndex.
-        if isinstance(self.axes[axis.value], pandas.MultiIndex) and isinstance(
-            new_axes[axis.value][0], tuple
-        ):
-            new_axes[axis.value] = pandas.MultiIndex.from_tuples(
-                new_axes[axis.value].values
-            )
+
         new_axes[axis.value] = new_axes[axis.value].set_names(
             self.axes[axis.value].names
         )
diff --git a/modin/core/dataframe/pandas/partitioning/partition_manager.py b/modin/core/dataframe/pandas/partitioning/partition_manager.py
--- a/modin/core/dataframe/pandas/partitioning/partition_manager.py
+++ b/modin/core/dataframe/pandas/partitioning/partition_manager.py
@@ -880,8 +880,11 @@ def get_indices(cls, axis, partitions, index_func=None):
         target = partitions.T if axis == 0 else partitions
         new_idx = [idx.apply(func) for idx in target[0]] if len(target) else []
         new_idx = cls.get_objects_from_partitions(new_idx)
-        # TODO FIX INFORMATION LEAK!!!!1!!1!!
-        total_idx = new_idx[0].append(new_idx[1:]) if new_idx else new_idx
+        # filter empty indexes
+        total_idx = list(filter(len, new_idx))
+        if len(total_idx) > 0:
+            # TODO FIX INFORMATION LEAK!!!!1!!1!!
+            total_idx = total_idx[0].append(total_idx[1:])
         return total_idx, new_idx
 
     @classmethod


Test patch:
Repository: modin-project/modin

Issue:
REFACTOR: use `pandas.util.cache_readonly` for `__constructors__`



Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -176,7 +176,7 @@ class PandasDataframe(ClassLogger):
     _deferred_index = False
     _deferred_column = False
 
-    @property
+    @pandas.util.cache_readonly
     def __constructor__(self):
         """
         Create a new instance of this object.
diff --git a/modin/core/storage_formats/base/query_compiler.py b/modin/core/storage_formats/base/query_compiler.py
--- a/modin/core/storage_formats/base/query_compiler.py
+++ b/modin/core/storage_formats/base/query_compiler.py
@@ -3225,7 +3225,7 @@ def write_items(df, broadcasted_items):
 
     # END Abstract methods for QueryCompiler
 
-    @property
+    @pandas.util.cache_readonly
     def __constructor__(self):
         """
         Get query compiler constructor.
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -70,7 +70,7 @@
 
 # Do not lookup certain attributes in columns or index, as they're used for some
 # special purposes, like serving remote context
-_ATTRS_NO_LOOKUP = {"____id_pack__", "__name__"}
+_ATTRS_NO_LOOKUP = {"____id_pack__", "__name__", "_cache"}
 
 _DEFAULT_BEHAVIOUR = {
     "__init__",
@@ -94,6 +94,7 @@
     "_reduce_dimension",
     "__repr__",
     "__len__",
+    "__constructor__",
     "_create_or_update_from_compiler",
     "_update_inplace",
     # for persistance support;
@@ -541,23 +542,17 @@ def _get_axis_number(cls, axis):
 
         return cls._pandas_class._get_axis_number(axis) if axis is not None else 0
 
-    def __constructor__(self, *args, **kwargs):
+    @pandas.util.cache_readonly
+    def __constructor__(self):
         """
         Construct DataFrame or Series object depending on self type.
 
-        Parameters
-        ----------
-        *args : list
-            Additional positional arguments to be passed to constructor.
-        **kwargs : dict
-            Additional keywords arguments to be passed to constructor.
-
         Returns
         -------
         modin.pandas.BasePandasDataset
             Constructed object.
         """
-        return type(self)(*args, **kwargs)
+        return type(self)
 
     def abs(self):  # noqa: RT01, D200
         """


Test patch:
Repository: modin-project/modin

Issue:
[ASV] port DatetimeAccessor and Categories benchmarks from pandas github
Have to add ASV benchmarks related which does the following actions
1) Accessor object for datetimelike
2) Setting categories
3) Removing categories


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/benchmarks.py b/asv_bench/benchmarks/benchmarks.py
--- a/asv_bench/benchmarks/benchmarks.py
+++ b/asv_bench/benchmarks/benchmarks.py
@@ -1068,4 +1068,49 @@ def time_drop_dups_string(self, shape):
         execute(self.series)
 
 
+class TimeDatetimeAccessor:
+
+    params = [get_benchmark_shapes("TimeDatetimeAccessor")]
+    param_names = ["shape"]
+
+    def setup(self, shape):
+        self.series = IMPL.Series(
+            IMPL.timedelta_range("1 days", periods=shape[0], freq="h")
+        )
+        execute(self.series)
+
+    def time_dt_accessor(self, shape):
+        execute(self.series.dt)
+
+    def time_timedelta_days(self, shape):
+        execute(self.series.dt.days)
+
+    def time_timedelta_seconds(self, shape):
+        execute(self.series.dt.seconds)
+
+
+class BaseCategories:
+    def setup(self, shape):
+        rows = shape[0]
+        arr = [f"s{i:04d}" for i in np.random.randint(0, rows // 10, size=rows)]
+        self.ts = IMPL.Series(arr).astype("category")
+        execute(self.ts)
+
+
+class TimeSetCategories(BaseCategories):
+    params = [get_benchmark_shapes("TimeSetCategories")]
+    param_names = ["shape"]
+
+    def time_set_categories(self, shape):
+        execute(self.ts.cat.set_categories(self.ts.cat.categories[::2]))
+
+
+class TimeRemoveCategories(BaseCategories):
+    params = [get_benchmark_shapes("TimeRemoveCategories")]
+    param_names = ["shape"]
+
+    def time_remove_categories(self, shape):
+        execute(self.ts.cat.remove_categories(self.ts.cat.categories[::2]))
+
+
 from .utils import setup  # noqa: E402, F401
diff --git a/asv_bench/benchmarks/utils/data_shapes.py b/asv_bench/benchmarks/utils/data_shapes.py
--- a/asv_bench/benchmarks/utils/data_shapes.py
+++ b/asv_bench/benchmarks/utils/data_shapes.py
@@ -130,6 +130,9 @@
             "TimeFillnaSeries",
             "TimeIndexingNumericSeries",
             "TimeFillnaMethodSeries",
+            "TimeDatetimeAccessor",
+            "TimeSetCategories",
+            "TimeRemoveCategories",
             "TimeDropDuplicatesSeries",
         ],
     ),


Test patch:
Repository: modin-project/modin

Issue:
FEAT: currently `rebalance_partitions` isn't enabled for `Unidist`
I see no reason why it shouldn't work with Unidist.


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/partitioning/partition_manager.py b/modin/core/dataframe/pandas/partitioning/partition_manager.py
--- a/modin/core/dataframe/pandas/partitioning/partition_manager.py
+++ b/modin/core/dataframe/pandas/partitioning/partition_manager.py
@@ -1386,7 +1386,10 @@ def rebalance_partitions(cls, partitions):
         list[int] or None
             Row lengths if possible to compute it.
         """
-        if Engine.get() in ["Ray", "Dask"] and StorageFormat.get() == "Pandas":
+        if (
+            Engine.get() in ["Ray", "Dask", "Unidist"]
+            and StorageFormat.get() == "Pandas"
+        ):
             # Rebalancing partitions is currently only implemented for PandasOnRay and PandasOnDask.
             # We rebalance when the ratio of the number of existing partitions to
             # the ideal number of partitions is larger than this threshold. The


Test patch:
diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -2025,7 +2025,7 @@ def test_groupby_with_virtual_partitions():
 
     # Check that the constructed Modin DataFrame has virtual partitions when
     # using Ray or Dask, and doesn't when using another execution engines.
-    if Engine.get() in ["Ray", "Dask"]:
+    if Engine.get() in ["Ray", "Dask", "Unidist"]:
         assert issubclass(
             type(big_modin_df._query_compiler._modin_frame._partitions[0][0]),
             PandasDataframeAxisPartition,
diff --git a/modin/test/test_partition_api.py b/modin/test/test_partition_api.py
--- a/modin/test/test_partition_api.py
+++ b/modin/test/test_partition_api.py
@@ -96,7 +96,7 @@ def get_df(lib, data):
         actual_axis_partitions = unwrap_partitions(df, axis=axis)
         assert len(expected_axis_partitions) == len(actual_axis_partitions)
         for item_idx in range(len(expected_axis_partitions)):
-            if Engine.get() in ["Ray", "Dask"]:
+            if Engine.get() in ["Ray", "Dask", "Unidist"]:
                 df_equals(
                     get_func(expected_axis_partitions[item_idx]),
                     get_func(actual_axis_partitions[item_idx]),
Repository: modin-project/modin

Issue:
REFACTOR: improve setup function in `TimeDropDuplicatesDataframe` bench
Assigning columns in a loop is very inefficient.


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/benchmarks.py b/asv_bench/benchmarks/benchmarks.py
--- a/asv_bench/benchmarks/benchmarks.py
+++ b/asv_bench/benchmarks/benchmarks.py
@@ -1032,13 +1032,17 @@ class TimeDropDuplicatesDataframe:
     param_names = ["shape"]
 
     def setup(self, shape):
+        from pandas import DataFrame
+
         rows, cols = shape
         N = rows // 10
         K = 10
-        self.df = IMPL.DataFrame()
+        # Assigning a large number of columns - inefficient in Modin, so use pandas
+        temp_df = DataFrame()
         # dataframe would  have cols-1 keys(strings) and one value(int) column
         for col in range(cols - 1):
-            self.df["key" + str(col + 1)] = tm.makeStringIndex(N).values.repeat(K)
+            temp_df["key" + str(col + 1)] = tm.makeStringIndex(N).values.repeat(K)
+        self.df = IMPL.DataFrame(temp_df)
         self.df["value"] = np.random.randn(N * K)
         execute(self.df)
 


Test patch:
Repository: modin-project/modin

Issue:
Consider increasing the minimal version of ray, in Modin environments.
Older versions of ray (`ray<1.13.0`) did not have a limitation on the protobuff version. They are compatible with `protobuff<4.0`. However, at the moment, a newer version of the protobuff is being installed into a environment, which leads to errors.

For example, I already tried to use [`ray==1.8.0`](https://github.com/modin-project/modin/actions/runs/3648941493/jobs/6163092982) and [`ray==1.11.0`](https://github.com/modin-project/modin/actions/runs/3649267522/jobs/6163851182).

Should we increase the minimum supported version of Ray (`>=1.4.0`), knowing that neither conda nor pip install dependencies that are compatible with each other? @modin-project/modin-core 

I propose to use `ray>=1.13.0` as it also fixes another problem (with `redis`) described in https://github.com/modin-project/modin/issues/4398.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -5,11 +5,7 @@
     long_description = fh.read()
 
 dask_deps = ["dask>=2.22.0", "distributed>=2.22.0"]
-ray_deps = [
-    "ray[default]>=1.4.0",
-    "pyarrow>=4.0.1",
-    "redis>=3.5.0,<4.0.0",
-]
+ray_deps = ["ray[default]>=1.13.0", "pyarrow"]
 unidist_deps = ["unidist[mpi]>=0.2.1"]
 remote_deps = ["rpyc==4.1.5", "cloudpickle", "boto3"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]


Test patch:
Repository: modin-project/modin

Issue:
Fix code scanning alert - Unused global variable
<!-- Warning: The suggested title contains the alert rule name. This can expose security information. -->

Tracking issue for:
- [ ] https://github.com/modin-project/modin/security/code-scanning/311



Discussion:

Resolving patch:
diff --git a/modin/pandas/indexing.py b/modin/pandas/indexing.py
--- a/modin/pandas/indexing.py
+++ b/modin/pandas/indexing.py
@@ -233,11 +233,6 @@ def boolean_mask_to_numeric(indexer):
 INCLUDED, END point is EXCLUDED), listlike of integers, boolean array] types.
 """
 
-_VIEW_IS_COPY_WARNING = """
-Modin is making a copy of of the DataFrame. This behavior diverges from Pandas.
-This will be fixed in future releases.
-"""
-
 
 def _compute_ndim(row_loc, col_loc):
     """


Test patch: