internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
Fix ASV after changing default branch: "master" -> "main"



Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/benchmarks.py b/asv_bench/benchmarks/benchmarks.py
--- a/asv_bench/benchmarks/benchmarks.py
+++ b/asv_bench/benchmarks/benchmarks.py
@@ -1029,20 +1029,17 @@ class TimeDropDuplicatesDataframe:
     param_names = ["shape"]
 
     def setup(self, shape):
-        from pandas import DataFrame
-
         rows, cols = shape
         N = rows // 10
         K = 10
-        # Assigning a large number of columns - inefficient in Modin, so use pandas
-        temp_df = DataFrame()
-        # dataframe would  have cols-1 keys(strings) and one value(int) column
+        data = {}
+        # dataframe would have cols-1 keys(strings) and one value(int) column
         for col in range(cols - 1):
-            temp_df["key" + str(col + 1)] = IMPL.Index(
+            data["key" + str(col + 1)] = IMPL.Index(
                 [f"i-{i}" for i in range(N)], dtype=object
             ).values.repeat(K)
-        self.df = IMPL.DataFrame(temp_df)
-        self.df["value"] = np.random.randn(N * K)
+        data["value"] = np.random.randn(N * K)
+        self.df = IMPL.DataFrame(data)
         execute(self.df)
 
     def time_drop_dups(self, shape):


Test patch:
Repository: modin-project/modin

Issue:
Update `dataframe-api-compat` version



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 # ray==2.5.0 broken: https://github.com/conda-forge/ray-packages-feedstock/issues/100
 ray_deps = ["ray[default]>=2.1.0,!=2.5.0", "pyarrow>=7.0.0"]
 mpi_deps = ["unidist[mpi]>=0.2.1"]
-consortium_standard_deps = ["dataframe-api-compat@git+https://github.com/data-apis/dataframe-api-compat.git@main"]
+consortium_standard_deps = ["dataframe-api-compat>=0.2.7"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]
 # Currently, Modin does not include `mpi` option in `all`.
 # Otherwise, installation of modin[all] would fail because


Test patch:
Repository: modin-project/modin

Issue:
Consider remove `use_legacy_dataset=False`
Reason: `FutureWarning: Passing 'use_legacy_dataset' is deprecated as of pyarrow 15.0.0 and will be removed in a future version.`.


Discussion:

Resolving patch:
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -187,9 +187,7 @@ class PyArrowDataset(ColumnStoreDataset):
     def _init_dataset(self):  # noqa: GL08
         from pyarrow.parquet import ParquetDataset
 
-        return ParquetDataset(
-            self.fs_path, filesystem=self.fs, use_legacy_dataset=False
-        )
+        return ParquetDataset(self.fs_path, filesystem=self.fs)
 
     @property
     def pandas_metadata(self):


Test patch:
Repository: modin-project/modin

Issue:
Do `right._modin_frame.combine()` for merge and join only when necessary



Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/merge.py b/modin/core/storage_formats/pandas/merge.py
--- a/modin/core/storage_formats/pandas/merge.py
+++ b/modin/core/storage_formats/pandas/merge.py
@@ -115,7 +115,6 @@ def row_axis_merge(cls, left, right, kwargs):
         left_index = kwargs.get("left_index", False)
         right_index = kwargs.get("right_index", False)
         sort = kwargs.get("sort", False)
-        right_to_broadcast = right._modin_frame.combine()
 
         if how in ["left", "inner"] and left_index is False and right_index is False:
             kwargs["sort"] = False
@@ -160,6 +159,7 @@ def map_func(
             elif on is not None:
                 on = list(on) if is_list_like(on) else [on]
 
+            right_to_broadcast = right._modin_frame.combine()
             new_columns, new_dtypes = cls._compute_result_metadata(
                 left, right, on, left_on, right_on, kwargs.get("suffixes", ("_x", "_y"))
             )
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -528,13 +528,13 @@ def join(self, right, **kwargs):
         on = kwargs.get("on", None)
         how = kwargs.get("how", "left")
         sort = kwargs.get("sort", False)
-        right_to_broadcast = right._modin_frame.combine()
 
         if how in ["left", "inner"]:
 
             def map_func(left, right, kwargs=kwargs):  # pragma: no cover
                 return pandas.DataFrame.join(left, right, **kwargs)
 
+            right_to_broadcast = right._modin_frame.combine()
             new_self = self.__constructor__(
                 self._modin_frame.broadcast_apply_full_axis(
                     axis=1,


Test patch:
Repository: modin-project/modin

Issue:
HDK deprecation
We stopped supporting HDK and [its repo](https://github.com/intel-ai/hdk) is archived. There are no plans to resume work on it in future. That's why, we are deprecating HDK engine and will remove it completely in a future release of Modin.


Discussion:

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -253,6 +253,26 @@ def _get_default(cls) -> str:
             "Please refer to installation documentation page to install an engine"
         )
 
+    @classmethod
+    def get(cls) -> str:
+        """
+        Get value of the Engine.
+
+        Returns
+        -------
+        str
+        """
+        value = super().get()
+        if value == "Native":
+            from modin.error_message import ErrorMessage
+
+            ErrorMessage.single_warning(
+                "HDK engine is deprecated and will be removed in a future version. "
+                + "Consider switching to Ray, Dask or MPI engine.",
+                FutureWarning,
+            )
+        return value
+
     @classmethod
     @doc(Parameter.add_option.__doc__)
     def add_option(cls, choice: Any) -> Any:
diff --git a/modin/error_message.py b/modin/error_message.py
--- a/modin/error_message.py
+++ b/modin/error_message.py
@@ -12,7 +12,7 @@
 # governing permissions and limitations under the License.
 
 import warnings
-from typing import NoReturn, Set
+from typing import NoReturn, Optional, Set
 
 from modin.logging import get_logger
 from modin.utils import get_current_execution
@@ -36,7 +36,11 @@ def not_implemented(cls, message: str = "") -> NoReturn:
         )
 
     @classmethod
-    def single_warning(cls, message: str) -> None:
+    def single_warning(
+        cls, message: str, category: Optional[type[Warning]] = None
+    ) -> None:
+        # note that there should not be identical messages with different categories since
+        # only the message is used as the hash key.
         message_hash = hash(message)
         logger = get_logger()
         if message_hash in cls.printed_warnings:
@@ -46,7 +50,7 @@ def single_warning(cls, message: str) -> None:
             return
 
         logger.debug(f"Modin Warning: Single Warning: {message} was raised.")
-        warnings.warn(message)
+        warnings.warn(message, category=category)
         cls.printed_warnings.add(message_hash)
 
     @classmethod


Test patch:
Repository: modin-project/modin

Issue:
Pin `pyarrow>=10.0.1` as pandas 2.2.* does
https://github.com/pandas-dev/pandas/blob/f538741432edf55c6b9fb5d0d496d2dd1d7c2457/pyproject.toml#L110


Discussion:

Resolving patch:
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -210,11 +210,7 @@ def row_groups_per_file(self):
 
     @functools.cached_property
     def files(self):
-        try:
-            files = self.dataset.files
-        except AttributeError:
-            # compatibility at least with 3.0.0 <= pyarrow < 8.0.0
-            files = self.dataset._dataset.files
+        files = self.dataset.files
         return self._get_files(files)
 
     def to_pandas_dataframe(
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 dask_deps = ["dask>=2.22.0", "distributed>=2.22.0"]
 # ray==2.5.0 broken: https://github.com/conda-forge/ray-packages-feedstock/issues/100
-ray_deps = ["ray>=2.1.0,!=2.5.0", "pyarrow>=7.0.0"]
+ray_deps = ["ray>=2.1.0,!=2.5.0", "pyarrow>=10.0.1"]
 mpi_deps = ["unidist[mpi]>=0.2.1"]
 consortium_standard_deps = ["dataframe-api-compat>=0.2.7"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]


Test patch:
Repository: modin-project/modin

Issue:
Remove read_pickle_distributed/to_pickle_distributed
They were [deprecated](https://github.com/modin-project/modin/pull/6957) for 0.28.0.


Discussion:
We also have a bunch of other methods like from_\*/to_\* we deprecated. Can we remove them in one go?
> We also have a bunch of other methods like from_*/to_* we deprecated. Can we remove them in one go?

We can. Do you have a list of such functions?

Resolving patch:
diff --git a/modin/experimental/pandas/__init__.py b/modin/experimental/pandas/__init__.py
--- a/modin/experimental/pandas/__init__.py
+++ b/modin/experimental/pandas/__init__.py
@@ -33,7 +33,6 @@
 """
 
 from modin.pandas import *  # noqa F401, F403
-from modin.utils import func_from_deprecated_location
 
 from .io import (  # noqa F401
     read_csv_glob,
@@ -44,10 +43,3 @@
     read_sql,
     read_xml_glob,
 )
-
-read_pickle_distributed = func_from_deprecated_location(
-    "read_pickle_glob",
-    "modin.experimental.pandas.io",
-    "`modin.experimental.pandas.read_pickle_distributed` is deprecated and will be removed in a future version. "
-    + "Please use `modin.experimental.pandas.to_pickle_glob` instead.",
-)
diff --git a/modin/pandas/accessor.py b/modin/pandas/accessor.py
--- a/modin/pandas/accessor.py
+++ b/modin/pandas/accessor.py
@@ -331,22 +331,6 @@ def to_pickle_glob(
             storage_options=storage_options,
         )
 
-    def to_pickle_distributed(
-        self,
-        filepath_or_buffer,
-        compression: CompressionOptions = "infer",
-        protocol: int = pickle.HIGHEST_PROTOCOL,
-        storage_options: StorageOptions = None,
-    ) -> None:  # noqa
-        warnings.warn(
-            "`DataFrame.modin.to_pickle_distributed` is deprecated and will be removed in a future version. "
-            + "Please use `DataFrame.modin.to_pickle_glob` instead.",
-            category=FutureWarning,
-        )
-        return self.to_pickle_glob(
-            filepath_or_buffer, compression, protocol, storage_options
-        )
-
     def to_parquet_glob(
         self,
         path,


Test patch:
diff --git a/modin/tests/experimental/test_io_exp.py b/modin/tests/experimental/test_io_exp.py
--- a/modin/tests/experimental/test_io_exp.py
+++ b/modin/tests/experimental/test_io_exp.py
@@ -253,8 +253,8 @@ def _pandas_read_csv_glob(path, storage_options):
 @pytest.mark.parametrize(
     "filename", ["test_default_to_pickle.pkl", "test_to_pickle*.pkl"]
 )
-@pytest.mark.parametrize("read_func", ["read_pickle_glob", "read_pickle_distributed"])
-@pytest.mark.parametrize("to_func", ["to_pickle_glob", "to_pickle_distributed"])
+@pytest.mark.parametrize("read_func", ["read_pickle_glob"])
+@pytest.mark.parametrize("to_func", ["to_pickle_glob"])
 def test_distributed_pickling(
     tmp_path, filename, compression, pathlike, read_func, to_func
 ):
Repository: modin-project/modin

Issue:
`enable_logging` decorator does not preserve decorated function type hints



Discussion:

Resolving patch:
diff --git a/modin/logging/logger_decorator.py b/modin/logging/logger_decorator.py
--- a/modin/logging/logger_decorator.py
+++ b/modin/logging/logger_decorator.py
@@ -17,9 +17,11 @@
 ``enable_logging`` is used for decorating individual Modin functions or classes.
 """
 
+from __future__ import annotations
+
 from functools import wraps
 from types import FunctionType, MethodType
-from typing import Any, Callable, Dict, Optional, Tuple, Type, Union
+from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, overload
 
 from modin.config import LogMode
 
@@ -27,8 +29,10 @@
 
 _MODIN_LOGGER_NOWRAP = "__modin_logging_nowrap__"
 
+Fn = TypeVar("Fn", bound=Any)
+
 
-def disable_logging(func: Callable) -> Callable:
+def disable_logging(func: Callable) -> Any:
     """
     Disable logging of one particular function. Useful for decorated classes.
 
@@ -46,11 +50,26 @@ def disable_logging(func: Callable) -> Callable:
     return func
 
 
+@overload
+def enable_logging(modin_layer: Fn) -> Fn:
+    # This helps preserve typings when the decorator is used without parentheses
+    pass
+
+
+@overload
+def enable_logging(
+    modin_layer: str = "PANDAS-API",
+    name: Optional[str] = None,
+    log_level: LogLevel = LogLevel.INFO,
+) -> Callable[[Fn], Fn]:
+    pass
+
+
 def enable_logging(
-    modin_layer: Union[str, Callable, Type] = "PANDAS-API",
+    modin_layer: str | Fn = "PANDAS-API",
     name: Optional[str] = None,
     log_level: LogLevel = LogLevel.INFO,
-) -> Callable:
+) -> Callable[[Fn], Fn] | Fn:
     """
     Log Decorator used on specific Modin functions or classes.
 
@@ -76,7 +95,7 @@ def enable_logging(
         # def func()
         return enable_logging()(modin_layer)
 
-    def decorator(obj: Any) -> Any:
+    def decorator(obj: Fn) -> Fn:
         """Decorate function or class to add logs to Modin API function(s)."""
         if isinstance(obj, type):
             seen: Dict[Any, Any] = {}
@@ -94,11 +113,11 @@ def decorator(obj: Any) -> Any:
                         )(attr_value)
 
                     setattr(obj, attr_name, wrapped)
-            return obj
+            return obj  # type: ignore [return-value]
         elif isinstance(obj, classmethod):
-            return classmethod(decorator(obj.__func__))
+            return classmethod(decorator(obj.__func__))  # type: ignore [return-value, arg-type]
         elif isinstance(obj, staticmethod):
-            return staticmethod(decorator(obj.__func__))
+            return staticmethod(decorator(obj.__func__))  # type: ignore [return-value, arg-type]
 
         assert isinstance(modin_layer, str), "modin_layer is somehow not a string!"
 


Test patch:
Repository: modin-project/modin

Issue:
how to take down ray and put up again in local mode
My program has memory risk, and part of it seems to come from memory leak (idling ray workers holding a big chunk of memory). I have a for loop to independently run chunks of csv file on a series of tasks, I wish to kill ray after each iteration to release memory, and let Modin to put it up again with fresh ray workers. However, my code is the following:
```py
import pandas

for df_ in pandas.read_csv('xxx.csv', chunk=5000):
    df_.to_csv(xxx)
    run_my_tasks(xxx) # Modin will initialize ray in first iteration
    ray.shutdown()
```

however, I got below error:
```
File "/home/.../lib/python3.9/site-packages/modin/core/execution/ray/common/deferred_execution.py", line 309, in _deconstruct_chain
    output[out_pos] = out_pos
IndexError: list assignment index out of range
```


Discussion:
Hi @SiRumCz, thanks for posting this issue. I guess there might be an issue with multiple Ray initialization in Modin codebase. We would have to look into this deeper. Meanwhile, can you explicitly put `ray.init()` before `run_my_tasks(xxx)` to see if it works?
@YarShev Thanks for your response. Yes, I have tried that method, and unfortunately I got:
`ValueError: An application is trying to access a Ray object whose owner is unknown(00ffffffffffffffffffffffffffffffffffffff0100000002e1f505). Please make sure that all Ray objects you are trying to access are part of the current Ray session. Note that object IDs generated randomly (ObjectID.from_random()) or out-of-band (ObjectID.from_binary(...)) cannot be passed as a task argument because Ray does not know which task created them. If this was not how your object ID was generated, please file an issue at https://github.com/ray-project/ray/issues/`
@SiRumCz, could you try to execute `ray.init()` and `importlib.reload(pd)` before `run_my_tasks(xxx)`, where `pd` is `import modin.pandas as pd`?


Resolving patch:
diff --git a/modin/utils.py b/modin/utils.py
--- a/modin/utils.py
+++ b/modin/utils.py
@@ -878,3 +878,17 @@ def __init__(self, func: Any):
 
     def __get__(self, instance: Any, owner: Any) -> Any:  # noqa: GL08
         return self.fget(owner)
+
+
+def reload_modin() -> None:
+    """
+    Reload all previously imported Modin modules.
+
+    The call to this function is required
+    if an execution engine has been shut down and
+    is going to be started up once again.
+    """
+    modules = sys.modules.copy()
+    for name, module in modules.items():
+        if name.startswith("modin"):
+            importlib.reload(module)


Test patch:
Repository: cornellius-gp/gpytorch

Issue:
RBF Kernel Change Breaks Testing Code
The change to RBFKernel in 84fccd898c45c08279fb5c109e6e234f3a47588a may break something about our prediction code. 

I am not totally sure what the problem is yet, but I isolated this as the problem with `git bisect` and have a reasonable test case where results are significantly worse with the commit in compared to after a revert commit. 

It seems like the stability issues we encountered when making this change in the past don't come up in the unit tests, but do on some real datasets.

I can try to push my test case to a branch as well, although it relies on a UCI dataset.

@Balandat @gpleiss 


Discussion:
I am not sure what the problem would be other than maybe some issue with how the bounds on the lengthscales are being handled. Did you adjust the bounds appropriately given the new definition? 

Can you post your test case?

Resolving patch:
diff --git a/gpytorch/kernels/rbf_kernel.py b/gpytorch/kernels/rbf_kernel.py
--- a/gpytorch/kernels/rbf_kernel.py
+++ b/gpytorch/kernels/rbf_kernel.py
@@ -3,6 +3,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import math
 from .kernel import Kernel
 
 
@@ -12,7 +13,7 @@ def __init__(
         self,
         ard_num_dims=None,
         log_lengthscale_bounds=(-10000, 10000),
-        eps=1e-5,
+        eps=1e-6,
         active_dims=None,
     ):
         super(RBFKernel, self).__init__(
@@ -24,6 +25,6 @@ def __init__(
         self.eps = eps
 
     def forward(self, x1, x2):
-        lengthscales = self.log_lengthscale.exp() + self.eps
+        lengthscales = self.log_lengthscale.exp().mul(math.sqrt(2)).clamp(self.eps, 1e5)
         diff = (x1.unsqueeze(2) - x2.unsqueeze(1)).div_(lengthscales)
-        return diff.pow_(2).sum(-1).mul_(-0.5).exp_()
+        return diff.pow_(2).sum(-1).mul_(-1).exp_()


Test patch:
Repository: cornellius-gp/gpytorch

Issue:
Possible extra derivative calculations at test time
See the discussion in #174. When calling backward to get the derivative of the mean function with respect to a test input, we enter the backward pass of `InvMatmul`. There doesn't seem to be an obvious reason we should be doing this, since none of the linear solves involved in the predictive __mean__ depend on the test point. 

This suggests we might also he computing derivatives with respect to model hyperparameters, which would be wasted computation.

@Balandat 


Discussion:
This might be solvable with a few judicious detach calls if we are in eval mode?
I think the cache's should be detached. This would probably solve the issue.

Resolving patch:
diff --git a/gpytorch/lazy/lazy_variable.py b/gpytorch/lazy/lazy_variable.py
--- a/gpytorch/lazy/lazy_variable.py
+++ b/gpytorch/lazy/lazy_variable.py
@@ -427,7 +427,7 @@ def exact_predictive_mean(self, full_mean, train_labels, n_train, likelihood, pr
         if res.ndimension() == 3:
             res = res.squeeze(-1)
         res = res + test_mean
-        return res, precomputed_cache
+        return res, precomputed_cache.detach()
 
     def _exact_predictive_covar_inv_quad_form_cache(self, train_train_covar_inv_root, test_train_covar):
         """
@@ -505,7 +505,7 @@ def exact_predictive_covar(self, n_train, likelihood, precomputed_cache=None):
 
         covar_inv_quad_form_root = self._exact_predictive_covar_inv_quad_form_root(precomputed_cache, test_train_covar)
         res = test_test_covar + RootLazyVariable(covar_inv_quad_form_root).mul(-1)
-        return res, precomputed_cache
+        return res, precomputed_cache.detach()
 
     def inv_matmul(self, tensor):
         """


Test patch:
Repository: cornellius-gp/gpytorch

Issue:
Repeated posterior evaluation (and backward) with same base_samples raises second backward issue
I am encountering an second backward error when doing repeated posterior evaluations with the same set of base_samples. If I use `detach_test_caches(True)` and `fast_pred_var()`, there is no issue. However, when I do not use `fast_pred_var()` and/or set `detach_test_caches(False)`, then the second backward issue is raised.

Repro:
[gpytorch_base_samples_caching_backward_issue.ipynb.txt](https://github.com/cornellius-gp/gpytorch/files/2937770/gpytorch_base_samples_caching_backward_issue.ipynb.txt)



Discussion:

Resolving patch:
diff --git a/gpytorch/lazy/lazy_tensor.py b/gpytorch/lazy/lazy_tensor.py
--- a/gpytorch/lazy/lazy_tensor.py
+++ b/gpytorch/lazy/lazy_tensor.py
@@ -995,7 +995,11 @@ def representation_tree(self):
 
     @property
     def requires_grad(self):
-        return any(arg.requires_grad for arg in tuple(self._args) + tuple(self._kwargs.values()))
+        return any(
+            arg.requires_grad
+            for arg in tuple(self._args) + tuple(self._kwargs.values())
+            if hasattr(arg, "requires_grad")
+        )
 
     @requires_grad.setter
     def requires_grad(self, val):
diff --git a/gpytorch/models/exact_prediction_strategies.py b/gpytorch/models/exact_prediction_strategies.py
--- a/gpytorch/models/exact_prediction_strategies.py
+++ b/gpytorch/models/exact_prediction_strategies.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import torch
+import contextlib
 
 from .. import settings
 from ..distributions import MultivariateNormal
@@ -315,9 +316,18 @@ def exact_predictive_covar(self, test_test_covar, test_train_covar):
             return ZeroLazyTensor(*test_test_covar.size())
 
         if settings.fast_pred_var.off():
-            train_train_covar = self.likelihood(
-                MultivariateNormal(torch.zeros(1), self.train_train_covar), self.train_inputs
-            ).lazy_covariance_matrix
+            if settings.detach_test_caches.on():
+                if self.train_train_covar.requires_grad:
+                    self.train_train_covar = self.train_train_covar.detach()
+                lik_context = torch.no_grad
+            else:
+                lik_context = contextlib.suppress
+
+            with lik_context():
+                train_train_covar = self.likelihood(
+                    MultivariateNormal(torch.zeros(1), self.train_train_covar), self.train_inputs
+                ).lazy_covariance_matrix
+
             test_train_covar = test_train_covar.evaluate()
             train_test_covar = test_train_covar.transpose(-1, -2)
             covar_correction_rhs = train_train_covar.inv_matmul(train_test_covar).mul(-1)


Test patch: