internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
[ASV] use `wait` function to get the right performance times



Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/scalability/scalability_benchmarks.py b/asv_bench/benchmarks/scalability/scalability_benchmarks.py
--- a/asv_bench/benchmarks/scalability/scalability_benchmarks.py
+++ b/asv_bench/benchmarks/scalability/scalability_benchmarks.py
@@ -60,4 +60,5 @@ def setup(self, shape, cpus):
         self.data = generate_dataframe("modin", "int", *shape, RAND_LOW, RAND_HIGH)
 
     def time_to_pandas(self, shape, cpus):
-        execute(to_pandas(self.data))
+        # to_pandas is already synchronous
+        to_pandas(self.data)
diff --git a/asv_bench/benchmarks/utils.py b/asv_bench/benchmarks/utils.py
--- a/asv_bench/benchmarks/utils.py
+++ b/asv_bench/benchmarks/utils.py
@@ -31,15 +31,17 @@
     NPARTITIONS = pd.DEFAULT_NPARTITIONS
 
 try:
-    from modin.config import TestDatasetSize, AsvImplementation
+    from modin.config import TestDatasetSize, AsvImplementation, Engine
 
     ASV_USE_IMPL = AsvImplementation.get()
     ASV_DATASET_SIZE = TestDatasetSize.get() or "Small"
+    ASV_USE_ENGINE = Engine.get()
 except ImportError:
     # The same benchmarking code can be run for different versions of Modin, so in
     # case of an error importing important variables, we'll just use predefined values
     ASV_USE_IMPL = os.environ.get("MODIN_ASV_USE_IMPL", "modin")
     ASV_DATASET_SIZE = os.environ.get("MODIN_TEST_DATASET_SIZE", "Small")
+    ASV_USE_ENGINE = os.environ.get("MODIN_ENGINE", "Ray")
 
 assert ASV_USE_IMPL in ("modin", "pandas")
 
@@ -217,7 +219,24 @@ def random_booleans(number):
 
 def execute(df):
     "Make sure the calculations are done."
-    return df.shape, df.dtypes
+    if ASV_USE_IMPL == "modin":
+        partitions = df._query_compiler._modin_frame._partitions
+        map(lambda partition: partition.drain_call_queue(), partitions)
+        if ASV_USE_ENGINE == "Ray":
+            from ray import wait
+
+            map(lambda partition: wait(partition.oid), partitions)
+        elif ASV_USE_ENGINE == "Dask":
+            from dask.distributed import wait
+
+            map(lambda partition: wait(partition.future), partitions)
+        elif ASV_USE_ENGINE == "Python":
+            pass
+
+    elif ASV_USE_IMPL == "pandas":
+        pass
+    else:
+        raise ValueError(f"wrong value of {ASV_USE_IMPL}")
 
 
 def get_shape_id(array):


Test patch:
Repository: modin-project/modin

Issue:
ModuleNotFoundError: No module named 'modin.distributed'
### Describe the problem
<!-- Describe the problem clearly here. -->
When installing Modin as follows `pip install git+https://github.com/modin-project/modin` and using partition API I got the error.

### Source code / logs
```python
from modin.distributed.dataframe.pandas import unwrap_partitions
ModuleNotFoundError: No module named 'modin.distributed'
```

We need to add init files in each subdirectory.



Discussion:

Resolving patch:
diff --git a/modin/distributed/__init__.py b/modin/distributed/__init__.py
new file mode 100644
--- /dev/null
+++ b/modin/distributed/__init__.py
@@ -0,0 +1,12 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
diff --git a/modin/distributed/dataframe/__init__.py b/modin/distributed/dataframe/__init__.py
new file mode 100644
--- /dev/null
+++ b/modin/distributed/dataframe/__init__.py
@@ -0,0 +1,12 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.


Test patch:
Repository: modin-project/modin

Issue:
Test on Ray Nightly
It would be great if we could test on the Ray nightly wheels on each commit to the master branch. I think we can add it as a separate CI and add a badge to the README to track.


Discussion:

Resolving patch:
diff --git a/modin/engines/ray/utils.py b/modin/engines/ray/utils.py
--- a/modin/engines/ray/utils.py
+++ b/modin/engines/ray/utils.py
@@ -153,18 +153,26 @@ def initialize_ray(
                     object_store_memory = None
             else:
                 object_store_memory = int(object_store_memory)
-            ray.init(
-                num_cpus=CpuCount.get(),
-                include_dashboard=False,
-                ignore_reinit_error=True,
-                _plasma_directory=plasma_directory,
-                object_store_memory=object_store_memory,
-                address=redis_address,
-                _redis_password=redis_password,
-                logging_level=100,
-                _memory=object_store_memory,
-                _lru_evict=True,
-            )
+
+            ray_init_kwargs = {
+                "num_cpus": CpuCount.get(),
+                "include_dashboard": False,
+                "ignore_reinit_error": True,
+                "_plasma_directory": plasma_directory,
+                "object_store_memory": object_store_memory,
+                "address": redis_address,
+                "_redis_password": redis_password,
+                "logging_level": 100,
+                "_memory": object_store_memory,
+                "_lru_evict": True,
+            }
+            from packaging import version
+
+            # setting of `_lru_evict` parameter raises DeprecationWarning since ray 2.0.0.dev0
+            if version.parse(ray.__version__) >= version.parse("2.0.0.dev0"):
+                ray_init_kwargs.pop("_lru_evict")
+            ray.init(**ray_init_kwargs)
+
         _move_stdlib_ahead_of_site_packages()
         ray.worker.global_worker.run_function_on_all_workers(
             _move_stdlib_ahead_of_site_packages


Test patch:
Repository: modin-project/modin

Issue:
Fix defensive assertions for Ray Client users
Related to https://github.com/ray-project/ray/issues/14042

For now, we will check for both types.


Discussion:
Note; this is in progress and will be fixed in a week or so 

Resolving patch:
diff --git a/modin/engines/ray/pandas_on_ray/frame/partition.py b/modin/engines/ray/pandas_on_ray/frame/partition.py
--- a/modin/engines/ray/pandas_on_ray/frame/partition.py
+++ b/modin/engines/ray/pandas_on_ray/frame/partition.py
@@ -20,11 +20,18 @@
 import ray
 from ray.worker import RayTaskError
 from ray.services import get_node_ip_address
+from packaging import version
+
+ObjectIDType = ray.ObjectRef
+if version.parse(ray.__version__) >= version.parse("1.2.0"):
+    from ray.util.client.common import ClientObjectRef
+
+    ObjectIDType = (ray.ObjectRef, ClientObjectRef)
 
 
 class PandasOnRayFramePartition(BaseFramePartition):
     def __init__(self, object_id, length=None, width=None, ip=None, call_queue=None):
-        assert type(object_id) is ray.ObjectRef
+        assert isinstance(object_id, ObjectIDType)
 
         self.oid = object_id
         if call_queue is None:
@@ -178,7 +185,7 @@ def length(self):
                 self._length_cache, self._width_cache = get_index_and_columns.remote(
                     self.oid
                 )
-        if isinstance(self._length_cache, ray.ObjectRef):
+        if isinstance(self._length_cache, ObjectIDType):
             try:
                 self._length_cache = ray.get(self._length_cache)
             except RayTaskError as e:
@@ -193,7 +200,7 @@ def width(self):
                 self._length_cache, self._width_cache = get_index_and_columns.remote(
                     self.oid
                 )
-        if isinstance(self._width_cache, ray.ObjectRef):
+        if isinstance(self._width_cache, ObjectIDType):
             try:
                 self._width_cache = ray.get(self._width_cache)
             except RayTaskError as e:
@@ -206,7 +213,7 @@ def ip(self):
                 self.drain_call_queue()
             else:
                 self._ip_cache = self.apply(lambda df: df)._ip_cache
-        if isinstance(self._ip_cache, ray.ObjectID):
+        if isinstance(self._ip_cache, ObjectIDType):
             try:
                 self._ip_cache = ray.get(self._ip_cache)
             except RayTaskError as e:
@@ -234,7 +241,7 @@ def get_index_and_columns(df):
 @ray.remote(num_returns=4)
 def deploy_ray_func(call_queue, partition):  # pragma: no cover
     def deserialize(obj):
-        if isinstance(obj, ray.ObjectRef):
+        if isinstance(obj, ObjectIDType):
             return ray.get(obj)
         return obj
 


Test patch:
Repository: modin-project/modin

Issue:
Benchmarks aren't trigger computations in case when MODIN_ENGINE=ray
This appears in case when `from modin.config import TestDatasetSize, AsvImplementation, Engine` ends with `ImportError` and user directly define environment variable as `export MODIN_ENGINE=ray`. But the rest of the code relies on the form where the first character is in uppercase.


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/utils.py b/asv_bench/benchmarks/utils.py
--- a/asv_bench/benchmarks/utils.py
+++ b/asv_bench/benchmarks/utils.py
@@ -43,16 +43,22 @@
     ASV_DATASET_SIZE = os.environ.get("MODIN_TEST_DATASET_SIZE", "Small")
     ASV_USE_ENGINE = os.environ.get("MODIN_ENGINE", "Ray")
 
+ASV_USE_IMPL = ASV_USE_IMPL.lower()
+ASV_DATASET_SIZE = ASV_DATASET_SIZE.lower()
+ASV_USE_ENGINE = ASV_USE_ENGINE.lower()
+
 assert ASV_USE_IMPL in ("modin", "pandas")
+assert ASV_DATASET_SIZE in ("big", "small")
+assert ASV_USE_ENGINE in ("ray", "dask", "python")
 
 BINARY_OP_DATA_SIZE = {
-    "Big": [
+    "big": [
         ((5000, 5000), (5000, 5000)),
         # the case extremely inefficient
         # ((20, 500_000), (10, 1_000_000)),
         ((500_000, 20), (1_000_000, 10)),
     ],
-    "Small": [
+    "small": [
         ((250, 250), (250, 250)),
         ((20, 10_000), (10, 25_000)),
         ((10_000, 20), (25_000, 10)),
@@ -60,13 +66,13 @@
 }
 
 UNARY_OP_DATA_SIZE = {
-    "Big": [
+    "big": [
         (5000, 5000),
         # the case extremely inefficient
         # (10, 1_000_000),
         (1_000_000, 10),
     ],
-    "Small": [
+    "small": [
         (250, 250),
         (10, 10_000),
         (10_000, 10),
@@ -74,8 +80,8 @@
 }
 
 GROUPBY_NGROUPS = {
-    "Big": [100, "huge_amount_groups"],
-    "Small": [5],
+    "big": [100, "huge_amount_groups"],
+    "small": [5],
 }
 
 IMPL = {
@@ -85,7 +91,7 @@
 
 
 def translator_groupby_ngroups(groupby_ngroups, shape):
-    if ASV_DATASET_SIZE == "Big":
+    if ASV_DATASET_SIZE == "big":
         if groupby_ngroups == "huge_amount_groups":
             return min(shape[0] // 2, 5000)
         return groupby_ngroups
@@ -227,21 +233,19 @@ def execute(df):
                 partitions.flatten(),
             )
         )
-        if ASV_USE_ENGINE == "Ray":
+        if ASV_USE_ENGINE == "ray":
             from ray import wait
 
             all(map(lambda partition: wait([partition.oid]), partitions.flatten()))
-        elif ASV_USE_ENGINE == "Dask":
+        elif ASV_USE_ENGINE == "dask":
             from dask.distributed import wait
 
             all(map(lambda partition: wait(partition.future), partitions.flatten()))
-        elif ASV_USE_ENGINE == "Python":
+        elif ASV_USE_ENGINE == "python":
             pass
 
     elif ASV_USE_IMPL == "pandas":
         pass
-    else:
-        raise ValueError(f"wrong value of {ASV_USE_IMPL}")
 
 
 def get_shape_id(array):


Test patch:
Repository: modin-project/modin

Issue:
Fix setting NPartitions via put
### Describe the problem
We need to reset `is_default` to `False` after base `put` call because we may meet `_get_default` call inside of base `put` (if `MODIN_NPARTITIONS` wasn't set) that can change `is_default` to `True`.

### Source code / logs
```python
import modin.pandas as pd
from modin.config import NPartitions
import numpy as np
NPartitions.put(1)
df = pd.DataFrame(np.arange(2 ** 10))
df._query_compiler._modin_frame._partitions.shape[0] == 1
False
```



Discussion:

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -154,8 +154,8 @@ class NPartitions(EnvironmentVariable, type=int):
 
     @classmethod
     def put(cls, value):
+        super().put(value)
         cls._is_default = False
-        return super().put(value)
 
     @classmethod
     def put_if_default(cls, value):


Test patch:
Repository: modin-project/modin

Issue:
[ASV] 'DataFrame.value_counts' benchmark is irrelevant
There are two problems in the current `DataFrame.value_counts` benchmark:
1. It uses `all` or `half` columns of the frame to count values on. That parameter forces us to group on thousands of columns for square-like or wide frames (we have `5000x5000` frame in our test set). This case is unrealistic and takes much time to execute, the proposal is to set fixed values for the `subset` parameter, for example: `[2, 10]`
2. Current benchmark implementation assumes, that all of the columns that was generated to group on is the first `N` columns of the frame:
https://github.com/modin-project/modin/blob/db784d55b17092c33794c3849cfe698f0cb9a7f7/asv_bench/benchmarks/benchmarks.py#L420
However, they're the last one) This also has to be fixed.

Changing the benchmark will erase all of the history of previous `value_counts` runs. But since all `value_counts` benchmarks are measurements of default to pandas implementation, this is not going to be a big loss.


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/benchmarks.py b/asv_bench/benchmarks/benchmarks.py
--- a/asv_bench/benchmarks/benchmarks.py
+++ b/asv_bench/benchmarks/benchmarks.py
@@ -394,30 +394,17 @@ def time_fillna(self, shape, limit, inplace):
 
 
 class BaseTimeValueCounts:
-    subset_params = {
-        "all": lambda shape: shape[1],
-        "half": lambda shape: shape[1] // 2,
-    }
-
-    def setup(self, shape, ngroups=5, subset="all"):
-        try:
-            subset = self.subset_params[subset]
-        except KeyError:
-            raise KeyError(
-                f"Invalid value for 'subset={subset}'. Allowed: {list(self.subset_params.keys())}"
-            )
-        ncols = subset(shape)
+    def setup(self, shape, ngroups=5, subset=1):
         ngroups = translator_groupby_ngroups(ngroups, shape)
-        self.df, _ = generate_dataframe(
+        self.df, self.subset = generate_dataframe(
             ASV_USE_IMPL,
             "int",
             *shape,
             RAND_LOW,
             RAND_HIGH,
-            groupby_ncols=ncols,
+            groupby_ncols=subset,
             count_groups=ngroups,
         )
-        self.subset = self.df.columns[:ncols].tolist()
 
 
 class TimeValueCountsFrame(BaseTimeValueCounts):
@@ -425,7 +412,7 @@ class TimeValueCountsFrame(BaseTimeValueCounts):
     params = [
         UNARY_OP_DATA_SIZE[ASV_DATASET_SIZE],
         GROUPBY_NGROUPS[ASV_DATASET_SIZE],
-        ["all", "half"],
+        [2, 10],
     ]
 
     def time_value_counts(self, *args, **kwargs):
@@ -442,7 +429,7 @@ class TimeValueCountsSeries(BaseTimeValueCounts):
 
     def setup(self, shape, ngroups, bins):
         super().setup(ngroups=ngroups, shape=shape)
-        self.df = self.df.iloc[:, 0]
+        self.df = self.df[self.subset[0]]
 
     def time_value_counts(self, shape, ngroups, bins):
         execute(self.df.value_counts(bins=bins))


Test patch:
Repository: modin-project/modin

Issue:
Ray APIs moved to private subpackage on latest master, breaking compatibility

### Describe the problem

Ray [recently moved](https://github.com/ray-project/ray/pull/14607) some of their internal APIs to an explicit `_private` subpackage. Modin calls one of these functions (`ray.utils.get_system_memory()`), leading to errors on tests that run on Ray's latest master.

### Source code / logs
[Example CI logs with failing tests](https://github.com/ray-project/xgboost_ray/pull/66/checks?check_run_id=2114689375)

```
                if object_store_memory is None:
                    # Round down to the nearest Gigabyte.
                    object_store_memory = int(
>                       0.6 * ray.utils.get_system_memory() // 10 ** 9 * 10 ** 9
                    )
E                   AttributeError: module 'ray' has no attribute 'utils'

/opt/hostedtoolcache/Python/3.6.9/x64/lib/python3.6/site-packages/modin/engines/ray/utils.py:132: AttributeError
```

<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/ray/utils.py b/modin/engines/ray/utils.py
--- a/modin/engines/ray/utils.py
+++ b/modin/engines/ray/utils.py
@@ -133,15 +133,21 @@ def initialize_ray(
                 # want to overwrite that value if we have.
                 if object_store_memory is None:
                     # Round down to the nearest Gigabyte.
-                    mem_bytes = ray.utils.get_system_memory() // 10 ** 9 * 10 ** 9
+                    try:
+                        system_memory = ray._private.utils.get_system_memory()
+                    except AttributeError:  # Compatibility with Ray <= 1.2
+                        system_memory = ray.utils.get_system_memory()
+                    mem_bytes = system_memory // 10 ** 9 * 10 ** 9
                     # Default to 8x memory for out of core
                     object_store_memory = 8 * mem_bytes
             # In case anything failed above, we can still improve the memory for Modin.
             if object_store_memory is None:
                 # Round down to the nearest Gigabyte.
-                object_store_memory = int(
-                    0.6 * ray.utils.get_system_memory() // 10 ** 9 * 10 ** 9
-                )
+                try:
+                    system_memory = ray._private.utils.get_system_memory()
+                except AttributeError:  # Compatibility with Ray <= 1.2
+                    system_memory = ray.utils.get_system_memory()
+                object_store_memory = int(0.6 * system_memory // 10 ** 9 * 10 ** 9)
                 # If the memory pool is smaller than 2GB, just use the default in ray.
                 if object_store_memory == 0:
                     object_store_memory = None


Test patch:
Repository: modin-project/modin

Issue:
`any` and `all` do not support `level=-1` on non-multi index
The following is not supported on Modin (but is supported in pandas):

```python
import modin.pandas as pd

df = pd.read_csv("some.csv", index_col=0)
df.any(level=-1)
df.all(level=-1)
```


Discussion:
Discovered as a part of #2847 

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -616,7 +616,7 @@ def all(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs):
                     )
                 if (
                     not self._query_compiler.has_multiindex(axis=axis)
-                    and level != 0
+                    and (level > 0 or level < -1)
                     and level != self.index.name
                 ):
                     raise ValueError(
@@ -671,7 +671,7 @@ def any(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs):
                     )
                 if (
                     not self._query_compiler.has_multiindex(axis=axis)
-                    and level != 0
+                    and (level > 0 or level < -1)
                     and level != self.index.name
                 ):
                     raise ValueError(


Test patch:
diff --git a/modin/pandas/test/dataframe/test_udf.py b/modin/pandas/test/dataframe/test_udf.py
--- a/modin/pandas/test/dataframe/test_udf.py
+++ b/modin/pandas/test/dataframe/test_udf.py
@@ -113,36 +113,7 @@ def test_apply_type_error(func):
 @pytest.mark.parametrize("axis", [0, 1])
 @pytest.mark.parametrize("level", [None, -1, 0, 1])
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
-@pytest.mark.parametrize(
-    "func",
-    [
-        "kurt",
-        pytest.param(
-            "count",
-            marks=pytest.mark.xfail(
-                reason="count method handle level parameter incorrectly"
-            ),
-        ),
-        pytest.param(
-            "sum",
-            marks=pytest.mark.xfail(
-                reason="sum method handle level parameter incorrectly"
-            ),
-        ),
-        pytest.param(
-            "mean",
-            marks=pytest.mark.xfail(
-                reason="mean method handle level parameter incorrectly"
-            ),
-        ),
-        pytest.param(
-            "all",
-            marks=pytest.mark.xfail(
-                reason="all method handle level parameter incorrectly"
-            ),
-        ),
-    ],
-)
+@pytest.mark.parametrize("func", ["kurt", "count", "sum", "mean", "all", "any"])
 def test_apply_text_func_with_level(level, data, func, axis):
     func_kwargs = {"level": level, "axis": axis}
     rows_number = len(next(iter(data.values())))  # length of the first data column
Repository: modin-project/modin

Issue:
Series `kurt` and `kurtosis` error messages do not match pandas
Discovered as a part of #2847


Discussion:

Resolving patch:
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -822,6 +822,14 @@ def iteritems(self):
     def keys(self):
         return self.index
 
+    def kurt(self, axis=None, skipna=None, level=None, numeric_only=None, **kwargs):
+        axis = self._get_axis_number(axis)
+        if numeric_only is True:
+            raise NotImplementedError("Series.kurt does not implement numeric_only.")
+        return super(Series, self).kurt(axis, skipna, level, numeric_only, **kwargs)
+
+    kurtosis = kurt
+
     def le(self, other, level=None, fill_value=None, axis=0):
         new_self, new_other = self._prepare_inter_op(other)
         return super(Series, new_self).le(new_other, level=level, axis=axis)


Test patch:
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -1964,19 +1964,7 @@ def test_kurtosis(axis, skipna):
 
 
 @pytest.mark.parametrize("axis", ["rows", "columns"])
-@pytest.mark.parametrize(
-    "numeric_only",
-    [
-        pytest.param(
-            True,
-            marks=pytest.mark.xfail(
-                reason="Modin - DID NOT RAISE <class 'NotImplementedError'>"
-            ),
-        ),
-        False,
-        None,
-    ],
-)
+@pytest.mark.parametrize("numeric_only", [True, False, None])
 def test_kurtosis_numeric_only(axis, numeric_only):
     eval_general(
         *create_test_series(test_data_diff_dtype),