internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
[REFACTOR] Concat for a single frame shouldn't cause additional operations
Currently, in OmniSci engine concat execution for a single frame adds an additional projection (which ais basically NOP). In some corner cases it may cause inefficiency (especially when API layer triggers frame execution). The esiest solution here is to just do nothing for a single frame concat case.


Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/frame/data.py b/modin/experimental/engines/omnisci_on_ray/frame/data.py
--- a/modin/experimental/engines/omnisci_on_ray/frame/data.py
+++ b/modin/experimental/engines/omnisci_on_ray/frame/data.py
@@ -562,6 +562,9 @@ def _union_all(
     def _concat(
         self, axis, other_modin_frames, join="outer", sort=False, ignore_index=False
     ):
+        if not other_modin_frames:
+            return self
+
         if axis == 0:
             return self._union_all(axis, other_modin_frames, join, sort, ignore_index)
 


Test patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
--- a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
+++ b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
@@ -480,6 +480,31 @@ def concat(lib, df1, df2):
 
         run_and_compare(concat, data=self.data, data2=self.data2, allow_subqueries=True)
 
+    @pytest.mark.parametrize("join", ["inner", "outer"])
+    @pytest.mark.parametrize("sort", bool_arg_values)
+    @pytest.mark.parametrize("ignore_index", bool_arg_values)
+    def test_concat_single(self, join, sort, ignore_index):
+        def concat(lib, df, join, sort, ignore_index):
+            return lib.concat([df], join=join, sort=sort, ignore_index=ignore_index)
+
+        run_and_compare(
+            concat,
+            data=self.data,
+            join=join,
+            sort=sort,
+            ignore_index=ignore_index,
+        )
+
+    def test_groupby_concat_single(self):
+        def concat(lib, df):
+            df = lib.concat([df])
+            return df.groupby("a").agg({"b": "min"})
+
+        run_and_compare(
+            concat,
+            data=self.data,
+        )
+
 
 class TestGroupby:
     data = {
Repository: modin-project/modin

Issue:
[OmniSci] Add float32 dtype support
Looks like our calcite serializer doesn't support float32 type.


Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py b/modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py
--- a/modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py
+++ b/modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py
@@ -41,6 +41,7 @@ class CalciteSerializer:
         "int32": "INTEGER",
         "int64": "BIGINT",
         "bool": "BOOLEAN",
+        "float32": "FLOAT",
         "float64": "DOUBLE",
     }
 


Test patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
--- a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
+++ b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
@@ -275,6 +275,20 @@ def test_sep_delimiter(self, kwargs):
 
         df_equals(modin_df, pandas_df)
 
+    @pytest.mark.skip(reason="https://github.com/modin-project/modin/issues/2174")
+    def test_float32(self):
+        csv_file = os.path.join(self.root, "modin/pandas/test/data", "test_usecols.csv")
+        kwargs = {
+            "dtype": {"a": "float32", "b": "float32"},
+        }
+
+        pandas_df = pandas.read_csv(csv_file, **kwargs)
+        pandas_df["a"] = pandas_df["a"] + pandas_df["b"]
+        modin_df = pd.read_csv(csv_file, **kwargs, engine="arrow")
+        modin_df["a"] = modin_df["a"] + modin_df["b"]
+
+        df_equals(modin_df, pandas_df)
+
 
 class TestMasks:
     data = {
Repository: modin-project/modin

Issue:
[Refactor]: Use python convention for first parameter name in methods.
Found by https://lgtm.com/projects/g/modin-project/modin?mode=list.


Discussion:

Resolving patch:
diff --git a/modin/experimental/backends/omnisci/query_compiler.py b/modin/experimental/backends/omnisci/query_compiler.py
--- a/modin/experimental/backends/omnisci/query_compiler.py
+++ b/modin/experimental/backends/omnisci/query_compiler.py
@@ -109,7 +109,7 @@ def view(self, index=None, columns=None):
         )
 
     def groupby_size(
-        query_compiler,
+        self,
         by,
         axis,
         groupby_args,
@@ -139,10 +139,10 @@ def groupby_size(
         -------
         BaseQueryCompiler
         """
-        new_frame = query_compiler._modin_frame.groupby_agg(
+        new_frame = self._modin_frame.groupby_agg(
             by,
             axis,
-            {query_compiler._modin_frame.columns[0]: "size"},
+            {self._modin_frame.columns[0]: "size"},
             groupby_args,
             **kwargs,
         )
@@ -152,12 +152,12 @@ def groupby_size(
         else:
             shape_hint = None
             new_frame = new_frame._set_columns(list(new_frame.columns)[:-1] + ["size"])
-        new_qc = query_compiler.__constructor__(new_frame, shape_hint=shape_hint)
+        new_qc = self.__constructor__(new_frame, shape_hint=shape_hint)
         if groupby_args["squeeze"]:
             new_qc = new_qc.squeeze()
         return new_qc
 
-    def groupby_sum(query_compiler, by, axis, groupby_args, map_args, **kwargs):
+    def groupby_sum(self, by, axis, groupby_args, map_args, **kwargs):
         """Groupby with sum aggregation.
 
         Parameters
@@ -177,15 +177,15 @@ def groupby_sum(query_compiler, by, axis, groupby_args, map_args, **kwargs):
         QueryCompiler
             A new QueryCompiler
         """
-        new_frame = query_compiler._modin_frame.groupby_agg(
+        new_frame = self._modin_frame.groupby_agg(
             by, axis, "sum", groupby_args, **kwargs
         )
-        new_qc = query_compiler.__constructor__(new_frame)
+        new_qc = self.__constructor__(new_frame)
         if groupby_args["squeeze"]:
             new_qc = new_qc.squeeze()
         return new_qc
 
-    def groupby_count(query_compiler, by, axis, groupby_args, map_args, **kwargs):
+    def groupby_count(self, by, axis, groupby_args, map_args, **kwargs):
         """Perform a groupby count.
 
         Parameters
@@ -209,10 +209,10 @@ def groupby_count(query_compiler, by, axis, groupby_args, map_args, **kwargs):
         -------
         QueryCompiler
         """
-        new_frame = query_compiler._modin_frame.groupby_agg(
+        new_frame = self._modin_frame.groupby_agg(
             by, axis, "count", groupby_args, **kwargs
         )
-        new_qc = query_compiler.__constructor__(new_frame)
+        new_qc = self.__constructor__(new_frame)
         if groupby_args["squeeze"]:
             new_qc = new_qc.squeeze()
         return new_qc


Test patch:
Repository: modin-project/modin

Issue:
Remove `PartitionIterator` methods related to the python 2.X iterator interface.
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`):
- **Python version**:
- **Code we can use to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/pandas/iterator.py b/modin/pandas/iterator.py
--- a/modin/pandas/iterator.py
+++ b/modin/pandas/iterator.py
@@ -43,9 +43,6 @@ def __iter__(self):
         return self
 
     def __next__(self):
-        return self.next()
-
-    def next(self):
         key = next(self.index_iter)
         df = self.df.iloc[key]
         return self.func(df)


Test patch:
Repository: modin-project/modin

Issue:
[BUG] [OmniSci] Float fragment size is passed to DBE
Looks like DBE eats it OK but we better pass proper integer values.


Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py b/modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py
--- a/modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py
+++ b/modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py
@@ -109,7 +109,7 @@ def put_arrow_to_omnisci(cls, table, name=None):
         if fragment_size is None:
             cpu_count = os.cpu_count()
             if cpu_count is not None:
-                fragment_size = table.num_rows / cpu_count
+                fragment_size = table.num_rows // cpu_count
                 fragment_size = min(fragment_size, 2 ** 25)
                 fragment_size = max(fragment_size, 2 ** 18)
             else:


Test patch:
Repository: modin-project/modin

Issue:
Add option for specify modin version in the cloud



Discussion:

Resolving patch:
diff --git a/modin/experimental/cloud/cluster.py b/modin/experimental/cloud/cluster.py
--- a/modin/experimental/cloud/cluster.py
+++ b/modin/experimental/cloud/cluster.py
@@ -251,7 +251,8 @@ def create(
     worker_node : str, optional
         What machine type to use for worker nodes in the cluster.
     add_conda_packages : list, optional
-        Custom conda packages for remote environments.
+        Custom conda packages for remote environments. By default remote modin version is
+        the same as local version.
     cluster_type : str, optional
         How to spawn the cluster.
         Currently spawning by Ray autoscaler ("rayscale" for general and "omnisci" for Omnisci-based) is supported
diff --git a/modin/experimental/cloud/rayscale.py b/modin/experimental/cloud/rayscale.py
--- a/modin/experimental/cloud/rayscale.py
+++ b/modin/experimental/cloud/rayscale.py
@@ -13,6 +13,7 @@
 
 import threading
 import os
+import re
 import traceback
 import sys
 from hashlib import sha1
@@ -148,7 +149,10 @@ def _conda_requirements(self):
 
         reqs.extend(self._get_python_version())
 
-        reqs.append(self._get_modin_version())
+        if not any(re.match(r"modin(\W|$)", p) for p in self.add_conda_packages):
+            # user didn't define modin release;
+            # use automatically detected modin release from local context
+            reqs.append(self._get_modin_version())
 
         if self.add_conda_packages:
             reqs.extend(self.add_conda_packages)


Test patch:
diff --git a/modin/experimental/cloud/test/test_cloud.py b/modin/experimental/cloud/test/test_cloud.py
--- a/modin/experimental/cloud/test/test_cloud.py
+++ b/modin/experimental/cloud/test/test_cloud.py
@@ -63,7 +63,8 @@ def ray_cluster():
             make_bootstrap_config_mock,
         ):
             ray_cluster = RayCluster(
-                Provider(name="aws"), add_conda_packages=["scikit-learn>=0.23"]
+                Provider(name="aws"),
+                add_conda_packages=["scikit-learn>=0.23", "modin==0.8.0"],
             )
         return ray_cluster
 
@@ -124,4 +125,5 @@ def test_update_conda_requirements(setup_commands_source, make_ray_cluster):
         in setup_commands_result
     )
     assert "scikit-learn>=0.23" in setup_commands_result
+    assert "modin==0.8.0" in setup_commands_result
     assert "{{CONDA_PACKAGES}}" not in setup_commands_result
Repository: modin-project/modin

Issue:
Support pandas 1.1.3



Discussion:
Should we do this now to try everything with newer version asap?

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -13,7 +13,7 @@
 
 import pandas
 
-__pandas_version__ = "1.1.2"
+__pandas_version__ = "1.1.3"
 
 if pandas.__version__ != __pandas_version__:
     import warnings
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -55,7 +55,7 @@ def is_pure(self):
     url="https://github.com/modin-project/modin",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=["pandas==1.1.2", "packaging"],
+    install_requires=["pandas==1.1.3", "packaging"],
     extras_require={
         # can be installed by pip install modin[dask]
         "dask": dask_deps,


Test patch:
diff --git a/modin/pandas/test/dataframe/test_binary.py b/modin/pandas/test/dataframe/test_binary.py
--- a/modin/pandas/test/dataframe/test_binary.py
+++ b/modin/pandas/test/dataframe/test_binary.py
@@ -150,16 +150,13 @@ def test_comparison(data, op, other):
     reason="Test is failing because of mismathing of thrown exceptions. See pandas issue #36377",
 )
 @pytest.mark.parametrize("other", ["a"])
-@pytest.mark.parametrize("op", ["ge", "gt", "le", "lt"])
+@pytest.mark.parametrize("op", ["ge", "gt", "le", "lt", "eq", "ne"])
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_comparison_except(data, op, other):
-    # So far `eq` and `ne` are excluded from testing because
-    # pandas throws an exception but Modin doesn't throw it.
-    with pytest.raises(AssertionError):
-        eval_general(
-            *create_test_dfs(data),
-            lambda df: getattr(df, op)(other),
-        )
+    eval_general(
+        *create_test_dfs(data),
+        lambda df: getattr(df, op)(other),
+    )
 
 
 @pytest.mark.parametrize("op", ["eq", "ge", "gt", "le", "lt", "ne"])
Repository: modin-project/modin

Issue:
Multiindex in modin give "slice unhashable error"
### System information
- MacOS
- 0.8.0
-3.7.6

```

import modin.pandas as pd
import numpy as np
index = pd.MultiIndex.from_product([np.arange(10), np.arange(10)], names=['f','s'])
data = np.arange(100)
df = pd.Series(data, index=index).sort_index()
df.loc[(slice(None), 1), ]

```


### Describe the problem
The code above gives `TypeError: unhashable type: 'slice'` error while the plain pandas runs without error




Discussion:
Hi @0luhancheng0, thanks for posting! I can reproduce this with what you've provided, it should be a straightforward fix. Thanks for the concise example!

Until we get this fixed, here is a workaround that will solve this issue:

```python
import modin.pandas as pd
import numpy as np
index = pd.MultiIndex.from_product([np.arange(10), np.arange(10)], names=['f','s'])
data = np.arange(100)
df = pd.Series(data, index=index).sort_index()
# df.loc[(slice(None), 1), ]
df.index = df.index.swaplevel()
df.loc[1,]
```

It's a silly workaround but it solves the case where we were trying to hash the slice until we get it fixed. Thanks again for posting this!

Resolving patch:
diff --git a/modin/pandas/indexing.py b/modin/pandas/indexing.py
--- a/modin/pandas/indexing.py
+++ b/modin/pandas/indexing.py
@@ -269,7 +269,11 @@ def __getitem__(self, key):
                 )
             ):
                 result.index = result.index.droplevel(list(range(len(col_loc))))
-            elif all(row_loc[i] in result.index.levels[i] for i in range(len(row_loc))):
+            elif all(
+                not isinstance(row_loc[i], slice)
+                and row_loc[i] in result.index.levels[i]
+                for i in range(len(row_loc))
+            ):
                 result.index = result.index.droplevel(list(range(len(row_loc))))
         if (
             hasattr(result, "columns")


Test patch:
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -2058,6 +2058,19 @@ def test_loc(data):
     pandas_result = pandas_series.loc[indices]
     df_equals(modin_result, pandas_result)
 
+    # From issue #1988
+    index = pd.MultiIndex.from_product([np.arange(10), np.arange(10)], names=["f", "s"])
+    data = np.arange(100)
+    modin_series = pd.Series(data, index=index).sort_index()
+    pandas_series = pandas.Series(data, index=index).sort_index()
+    modin_result = modin_series.loc[
+        (slice(None), 1),
+    ]
+    pandas_result = pandas_series.loc[
+        (slice(None), 1),
+    ]
+    df_equals(modin_result, pandas_result)
+
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_lt(data):
Repository: modin-project/modin

Issue:
Default Ray password length is insufficient
### Describe the problem
Modin generates a 16-byte Ray password which is too short. The recommendation is to have at least 32-byte secret

### Source code / logs
https://github.com/modin-project/modin/blob/cc57954466804bd7363ac4f5fe4d9967f7a1a169/modin/engines/ray/utils.py#L102



Discussion:

Resolving patch:
diff --git a/modin/engines/ray/utils.py b/modin/engines/ray/utils.py
--- a/modin/engines/ray/utils.py
+++ b/modin/engines/ray/utils.py
@@ -99,7 +99,7 @@ def initialize_ray(
 
         cluster = override_is_cluster or IsRayCluster.get()
         redis_address = override_redis_address or RayRedisAddress.get()
-        redis_password = override_redis_password or secrets.token_hex(16)
+        redis_password = override_redis_password or secrets.token_hex(32)
 
         if cluster:
             # We only start ray in a cluster setting for the head node.


Test patch:
Repository: modin-project/modin

Issue:
Support DataFrame.mean in OmniSci backend



Discussion:

Resolving patch:
diff --git a/modin/experimental/backends/omnisci/query_compiler.py b/modin/experimental/backends/omnisci/query_compiler.py
--- a/modin/experimental/backends/omnisci/query_compiler.py
+++ b/modin/experimental/backends/omnisci/query_compiler.py
@@ -289,22 +289,29 @@ def groupby_dict_agg(self, by, func_dict, groupby_args, agg_args, drop=False):
         )
         return self.__constructor__(new_frame)
 
-    def count(self, axis=0, level=None, **kwargs):
-        return self._agg("count")
+    def count(self, **kwargs):
+        return self._agg("count", **kwargs)
 
-    def max(self, axis=0, level=None, **kwargs):
-        return self._agg("max")
+    def max(self, **kwargs):
+        return self._agg("max", **kwargs)
 
-    def min(self, axis=0, level=None, **kwargs):
-        return self._agg("min")
+    def min(self, **kwargs):
+        return self._agg("min", **kwargs)
 
-    def sum(self, axis=0, level=None, **kwargs):
-        return self._agg("sum")
+    def sum(self, **kwargs):
+        return self._agg("sum", **kwargs)
+
+    def mean(self, **kwargs):
+        return self._agg("mean", **kwargs)
 
     def _agg(self, agg, axis=0, level=None, **kwargs):
         if level is not None or axis != 0:
             return getattr(super(), agg)(axis=axis, level=level, **kwargs)
 
+        skipna = kwargs.get("skipna", True)
+        if not skipna:
+            return getattr(super(), agg)(axis=axis, level=level, **kwargs)
+
         new_frame = self._modin_frame.agg(agg)
         new_frame = new_frame._set_index(
             pandas.Index.__new__(pandas.Index, data=["__reduced__"], dtype="O")


Test patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
--- a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
+++ b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
@@ -852,12 +852,19 @@ class TestAgg:
         "d": [11, 22, 33, 22, 33, 22],
     }
 
-    @pytest.mark.parametrize("agg", ["count", "max", "min", "sum"])
-    def test_simple_agg(self, agg):
-        def agg(df, agg, **kwargs):
-            return getattr(df, agg)()
+    @pytest.mark.parametrize("agg", ["max", "min", "sum", "mean"])
+    @pytest.mark.parametrize("skipna", bool_arg_values)
+    def test_simple_agg(self, agg, skipna):
+        def apply(df, agg, skipna, **kwargs):
+            return getattr(df, agg)(skipna=skipna)
 
-        run_and_compare(agg, data=self.data, agg=agg, force_lazy=False)
+        run_and_compare(apply, data=self.data, agg=agg, skipna=skipna, force_lazy=False)
+
+    def test_count_agg(self):
+        def apply(df, **kwargs):
+            return df.count()
+
+        run_and_compare(apply, data=self.data, force_lazy=False)
 
     @pytest.mark.parametrize("cols", ["a", "d"])
     @pytest.mark.parametrize("dropna", [True, False])
Repository: modin-project/modin

Issue:
Update dask dependencies according to requirements.txt
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`):
- **Python version**:
- **Code we can use to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -37,7 +37,7 @@ def is_pure(self):
         return False
 
 
-dask_deps = ["dask>=2.12.0", "distributed>=2.12.0"]
+dask_deps = ["dask>=2.12.0,<=2.19.0", "distributed>=2.12.0,<=2.19.0"]
 ray_deps = ["ray>=1.0.0", "pyarrow<0.17"]
 remote_deps = ["rpyc==4.1.5", "cloudpickle==1.4.1", "boto3==1.4.8"]
 


Test patch:
Repository: modin-project/modin

Issue:
Move import s3fs in parquet_reader.py in function scope
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`):
- **Python version**:
- **Code we can use to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

`s3fs` - optional dependence, so we shouldn't import in global scope.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/column_stores/parquet_reader.py b/modin/engines/base/io/column_stores/parquet_reader.py
--- a/modin/engines/base/io/column_stores/parquet_reader.py
+++ b/modin/engines/base/io/column_stores/parquet_reader.py
@@ -12,7 +12,6 @@
 # governing permissions and limitations under the License.
 
 import os
-import s3fs
 
 from modin.engines.base.io.column_stores.column_store_reader import ColumnStoreReader
 from modin.error_message import ErrorMessage
@@ -73,6 +72,8 @@ def _read(cls, path, engine, columns, **kwargs):
         else:
             directory = False
         if not columns:
+            import s3fs
+
             if directory:
                 # Path of the sample file that we will read to get the remaining columns
                 pd = ParquetDataset(path)


Test patch: