internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
ASV: add value_counts benchmark for OmniSci backend (dataframe case)



Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/omnisci/benchmarks.py b/asv_bench/benchmarks/omnisci/benchmarks.py
--- a/asv_bench/benchmarks/omnisci/benchmarks.py
+++ b/asv_bench/benchmarks/omnisci/benchmarks.py
@@ -244,25 +244,43 @@ def create_fillna_value(value_type: str, columns: list):
         return value
 
 
-class TimeValueCountsSeries:
-    param_names = ["shape", "ngroups"]
-    params = [
-        get_benchmark_shapes("omnisci.TimeValueCountsSeries"),
-        GROUPBY_NGROUPS,
-    ]
-
-    def setup(self, shape, ngroups):
+class BaseTimeValueCounts:
+    def setup(self, shape, ngroups=5, subset=1):
         ngroups = translator_groupby_ngroups(ngroups, shape)
-        self.df, self.column_names = generate_dataframe(
+        self.df, self.subset = generate_dataframe(
             ASV_USE_IMPL,
             "int",
             *shape,
             RAND_LOW,
             RAND_HIGH,
-            groupby_ncols=1,
+            groupby_ncols=subset,
             count_groups=ngroups,
         )
-        self.series = self.df[self.column_names[0]]
+        trigger_import(self.df)
+
+
+class TimeValueCountsDataFrame(BaseTimeValueCounts):
+    param_names = ["shape", "ngroups", "subset"]
+    params = [
+        get_benchmark_shapes("omnisci.TimeValueCountsDataFrame"),
+        GROUPBY_NGROUPS,
+        [2, 10],
+    ]
+
+    def time_value_counts(self, *args, **kwargs):
+        execute(self.df.value_counts(subset=self.subset))
+
+
+class TimeValueCountsSeries(BaseTimeValueCounts):
+    param_names = ["shape", "ngroups"]
+    params = [
+        get_benchmark_shapes("omnisci.TimeValueCountsSeries"),
+        GROUPBY_NGROUPS,
+    ]
+
+    def setup(self, shape, ngroups):
+        super().setup(shape, ngroups, subset=1)
+        self.series = self.df[self.subset[0]]
         trigger_import(self.series)
 
     def time_value_counts(self, shape, ngroups):
diff --git a/asv_bench/benchmarks/utils/data_shapes.py b/asv_bench/benchmarks/utils/data_shapes.py
--- a/asv_bench/benchmarks/utils/data_shapes.py
+++ b/asv_bench/benchmarks/utils/data_shapes.py
@@ -148,6 +148,7 @@
             "omnisci.TimeProperties",
             "omnisci.TimeGroupByDefaultAggregations",
             "omnisci.TimeGroupByMultiColumn",
+            "omnisci.TimeValueCountsDataFrame",
             # OmniSci backend IO benchmarks
             "omnisci.TimeReadCsvNames",
         ],


Test patch:
Repository: modin-project/modin

Issue:
JSON dispatcher data file split correction
Originated from https://github.com/modin-project/modin/pull/2607#discussion_r571989125.


Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/text/json_dispatcher.py b/modin/engines/base/io/text/json_dispatcher.py
--- a/modin/engines/base/io/text/json_dispatcher.py
+++ b/modin/engines/base/io/text/json_dispatcher.py
@@ -17,7 +17,6 @@
 from io import BytesIO
 import pandas
 import numpy as np
-from csv import QUOTE_NONE
 
 from modin.config import NPartitions
 
@@ -73,7 +72,6 @@ def _read(cls, path_or_buf, **kwargs):
             splits = cls.partitioned_file(
                 f,
                 num_partitions=NPartitions.get(),
-                is_quoting=(args.get("quoting", "") != QUOTE_NONE),
             )
             for start, end in splits:
                 args.update({"start": start, "end": end})


Test patch:
Repository: modin-project/modin

Issue:
Do not check ASV benchmarks on test data, where the number of rows is much less than the number of columns
These sizes can be removed because such cases are not used in benchmarking: https://github.com/modin-project/modin/blob/dd91a78ad3f4b8e3e569215e9c8e540ad099d4a8/asv_bench/benchmarks/utils/data_shapes.py#L33 and https://github.com/modin-project/modin/blob/dd91a78ad3f4b8e3e569215e9c8e540ad099d4a8/asv_bench/benchmarks/utils/data_shapes.py#L46


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/utils/data_shapes.py b/asv_bench/benchmarks/utils/data_shapes.py
--- a/asv_bench/benchmarks/utils/data_shapes.py
+++ b/asv_bench/benchmarks/utils/data_shapes.py
@@ -30,7 +30,6 @@
     ],
     "small": [
         [[250, 250], [250, 250]],
-        [[20, 10_000], [10, 25_000]],
         [[10_000, 20], [25_000, 10]],
     ],
 }
@@ -43,7 +42,6 @@
     ],
     "small": [
         [250, 250],
-        [10, 10_000],
         [10_000, 10],
     ],
 }


Test patch:
Repository: modin-project/modin

Issue:
Make parameter `label` of Modin DMatrix optional
### Describe the problem
<!-- Describe the problem clearly here. -->

To support drop-in-replacement API for DMatrix


Discussion:

Resolving patch:
diff --git a/modin/experimental/xgboost/xgboost.py b/modin/experimental/xgboost/xgboost.py
--- a/modin/experimental/xgboost/xgboost.py
+++ b/modin/experimental/xgboost/xgboost.py
@@ -35,7 +35,7 @@ class DMatrix:
     ----------
     data : modin.pandas.DataFrame
         Data source of DMatrix.
-    label : modin.pandas.DataFrame or modin.pandas.Series
+    label : modin.pandas.DataFrame or modin.pandas.Series, optional
         Labels used for training.
 
     Notes
@@ -43,16 +43,20 @@ class DMatrix:
     Currently DMatrix supports only `data` and `label` parameters.
     """
 
-    def __init__(self, data, label):
+    def __init__(self, data, label=None):
         assert isinstance(
             data, pd.DataFrame
         ), f"Type of `data` is {type(data)}, but expected {pd.DataFrame}."
-        assert isinstance(
-            label, (pd.DataFrame, pd.Series)
-        ), f"Type of `data` is {type(label)}, but expected {pd.DataFrame} or {pd.Series}."
+
+        if label is not None:
+            assert isinstance(
+                label, (pd.DataFrame, pd.Series)
+            ), f"Type of `data` is {type(label)}, but expected {pd.DataFrame} or {pd.Series}."
+            self.label = unwrap_partitions(label, axis=0)
+        else:
+            self.label = None
 
         self.data = unwrap_partitions(data, axis=0, get_ip=True)
-        self.label = unwrap_partitions(label, axis=0)
 
         self.metadata = (
             data.index,


Test patch:
Repository: modin-project/modin

Issue:
ASV: all shapes should be of list type
There are two ways to define shapes, by default through a dictionary or through a `json` file. Due to the fact that the `json` format does not support the use of `tuples`, we have to use the `list` type. But since in both cases the hash from the benchmark source code is considered and it must be the same, then the `list` type must also be used in dictionaries.

Here the shape are of `tuple` type: https://github.com/modin-project/modin/blob/600fa2632973167176333cb03c8a8c3b278be4ca/asv_bench/benchmarks/utils/data_shapes.py#L50
https://github.com/modin-project/modin/blob/600fa2632973167176333cb03c8a8c3b278be4ca/asv_bench/benchmarks/utils/data_shapes.py#L53


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/utils/data_shapes.py b/asv_bench/benchmarks/utils/data_shapes.py
--- a/asv_bench/benchmarks/utils/data_shapes.py
+++ b/asv_bench/benchmarks/utils/data_shapes.py
@@ -47,10 +47,10 @@
 }
 SERIES_DATA_SIZE = {
     "big": [
-        (100_000, 1),
+        [100_000, 1],
     ],
     "small": [
-        (10_000, 1),
+        [10_000, 1],
     ],
 }
 


Test patch:
Repository: modin-project/modin

Issue:
Use of deprecated `ray.services` package.
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`):
- **Python version**:
- **Code we can use to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
Modin still uses the deprecated `ray.services` package, which will be removed soon: https://github.com/ray-project/ray/pull/18475

E.g. https://github.com/modin-project/modin/blob/master/modin/engines/ray/pandas_on_ray/frame/partition.py#L23

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/ray/pandas_on_ray/frame/axis_partition.py b/modin/engines/ray/pandas_on_ray/frame/axis_partition.py
--- a/modin/engines/ray/pandas_on_ray/frame/axis_partition.py
+++ b/modin/engines/ray/pandas_on_ray/frame/axis_partition.py
@@ -19,7 +19,7 @@
 from .partition import PandasOnRayFramePartition
 
 import ray
-from ray.services import get_node_ip_address
+from ray.util import get_node_ip_address
 
 
 class PandasOnRayFrameAxisPartition(PandasFrameAxisPartition):
diff --git a/modin/engines/ray/pandas_on_ray/frame/partition.py b/modin/engines/ray/pandas_on_ray/frame/partition.py
--- a/modin/engines/ray/pandas_on_ray/frame/partition.py
+++ b/modin/engines/ray/pandas_on_ray/frame/partition.py
@@ -20,7 +20,7 @@
 from modin.pandas.indexing import compute_sliced_len
 
 import ray
-from ray.services import get_node_ip_address
+from ray.util import get_node_ip_address
 from packaging import version
 
 ObjectIDType = ray.ObjectRef
diff --git a/modin/experimental/xgboost/xgboost_ray.py b/modin/experimental/xgboost/xgboost_ray.py
--- a/modin/experimental/xgboost/xgboost_ray.py
+++ b/modin/experimental/xgboost/xgboost_ray.py
@@ -27,7 +27,7 @@
 import numpy as np
 import xgboost as xgb
 import ray
-from ray.services import get_node_ip_address
+from ray.util import get_node_ip_address
 import pandas
 
 from modin.distributed.dataframe.pandas import from_partitions


Test patch:
Repository: modin-project/modin

Issue:
Warn user about heterogeneous data presence during `read_csv` call and propose workaround
Subpart of https://github.com/modin-project/modin/issues/3278.



Discussion:

Resolving patch:
diff --git a/modin/backends/pandas/parsers.py b/modin/backends/pandas/parsers.py
--- a/modin/backends/pandas/parsers.py
+++ b/modin/backends/pandas/parsers.py
@@ -149,18 +149,36 @@ def get_dtypes(cls, dtypes_ids):
 
         Returns
         -------
-        pandas.Series
-            pandas.Series where index is columns names and values are
-            columns dtypes.
+        frame_dtypes : pandas.Series or dtype
+            Resulting dtype or pandas.Series where column names are used as
+            index and types of columns are used as values for full resulting
+            frame.
         """
+        # each element in `partitions_dtypes` is a Series, where column names are
+        # used as index and types of columns for different partitions are used as values
         partitions_dtypes = cls.materialize(dtypes_ids)
         if all([len(dtype) == 0 for dtype in partitions_dtypes]):
             return None
-        return (
-            pandas.concat(partitions_dtypes, axis=1)
-            .apply(lambda row: find_common_type_cat(row.values), axis=1)
-            .squeeze(axis=0)
-        )
+
+        combined_part_dtypes = pandas.concat(partitions_dtypes, axis=1)
+        frame_dtypes = combined_part_dtypes.iloc[:, 0]
+
+        if not combined_part_dtypes.eq(frame_dtypes, axis=0).all(axis=None):
+            ErrorMessage.missmatch_with_pandas(
+                operation="read_*",
+                message="Data types of partitions are different! "
+                "Please refer to the troubleshooting section of the Modin documentation "
+                "to fix this issue",
+            )
+
+            # concat all elements of `partitions_dtypes` and find common dtype
+            # for each of the column among all partitions
+            frame_dtypes = combined_part_dtypes.apply(
+                lambda row: find_common_type_cat(row.values),
+                axis=1,
+            ).squeeze(axis=0)
+
+        return frame_dtypes
 
     @classmethod
     def single_worker_read(cls, fname, **kwargs):


Test patch:
Repository: modin-project/modin

Issue:
Remove inheritance of Modin DMatrix from xgb.DMatrix
Inheritance of Modin DMatrix from xgb.DMatrix doesn't include any benefits. Wrong documentation is provided to user using `help(modin.experimtenal.xgboost.DMatrix)` command.


Discussion:

Resolving patch:
diff --git a/modin/experimental/xgboost/xgboost.py b/modin/experimental/xgboost/xgboost.py
--- a/modin/experimental/xgboost/xgboost.py
+++ b/modin/experimental/xgboost/xgboost.py
@@ -25,7 +25,7 @@
 LOGGER = logging.getLogger("[modin.xgboost]")
 
 
-class DMatrix(xgb.DMatrix):
+class DMatrix:
     """
     DMatrix holds references to partitions of Modin DataFrame.
 


Test patch:
Repository: modin-project/modin

Issue:
[OmniSci] Fix assert on empty record batch
We currently assert when OmniSci returns an empty arrow record batch.


Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_native/frame/partition_manager.py b/modin/experimental/engines/omnisci_on_native/frame/partition_manager.py
--- a/modin/experimental/engines/omnisci_on_native/frame/partition_manager.py
+++ b/modin/experimental/engines/omnisci_on_native/frame/partition_manager.py
@@ -252,7 +252,7 @@ def run_exec_plan(cls, plan, index_cols, dtypes, columns):
         curs = omniSession.executeRA(cmd_prefix + calcite_json)
         assert curs
         rb = curs.getArrowRecordBatch()
-        assert rb
+        assert rb is not None
         at = pyarrow.Table.from_batches([rb])
 
         res = np.empty((1, 1), dtype=np.dtype(object))


Test patch:
diff --git a/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py b/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
--- a/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
+++ b/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
@@ -1709,6 +1709,15 @@ def cmp(df, cmp_fn, value, **kwargs):
 
         run_and_compare(cmp, data=data, cmp_fn=cmp_fn, value=value)
 
+    @pytest.mark.xfail(
+        reason="Requires fix in OmniSci: https://github.com/intel-ai/omniscidb/pull/178"
+    )
+    def test_filter_empty_result(self):
+        def filter(df, **kwargs):
+            return df[df.a < 0]
+
+        run_and_compare(filter, data=self.cmp_data)
+
 
 class TestDateTime:
     datetime_data = {
Repository: modin-project/modin

Issue:
Unpin `mindsdb_sql` as soon as `dfsql` fixed own deps on `mindsdb_sql`
Revert changes, included by this #3248 



Discussion:
Did we open an issue on `dfsql`?
They are currently working on it.
https://github.com/mindsdb/dfsql/pull/19

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ def is_pure(self):
 ray_deps = ["ray[default]>=1.4.0", "pyarrow>=1.0"]
 remote_deps = ["rpyc==4.1.5", "cloudpickle", "boto3==1.4.8"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]
-sql_deps = ["dfsql>=0.2.6", "mindsdb_sql<=0.0.12"]
+sql_deps = ["dfsql>=0.4.2"]
 all_deps = dask_deps + ray_deps + remote_deps + spreadsheet_deps
 
 # dfsql does not support Windows yet


Test patch:
Repository: modin-project/modin

Issue:
[OmniSci] With OmniSci backed dtype property may include types for index
```
import modin.pandas as pd

df = pd.DataFrame(
    {
        "f1": [0, 5, 10, 15],
        "f2": [1000, 0, 2000, 1],
        "s": ["abc"] * 4,
        "i": [10] * 4,
        "t": [pd.to_datetime("2021")] * 4,
    }
)

df2 = df[(df.f1 < 10) & (df.f2 > 300)]
print(df2.dtypes)
```
leads to 
```
__index__             int64
f1                    int64
f2                    int64
s                    object
i                     int64
t            datetime64[ns]
```


Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_native/frame/data.py b/modin/experimental/engines/omnisci_on_native/frame/data.py
--- a/modin/experimental/engines/omnisci_on_native/frame/data.py
+++ b/modin/experimental/engines/omnisci_on_native/frame/data.py
@@ -2002,6 +2002,20 @@ def _get_columns(self):
     columns = property(_get_columns)
     index = property(_get_index)
 
+    @property
+    def dtypes(self):
+        """
+        Return column data types.
+
+        Returns
+        -------
+        pandas.Series
+            A pandas Series containing the data types for this dataframe.
+        """
+        if self._index_cols is not None:
+            return self._dtypes[len(self._index_cols) :]
+        return self._dtypes
+
     def has_multiindex(self):
         """
         Check for multi-index usage.


Test patch:
diff --git a/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py b/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
--- a/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
+++ b/modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
@@ -1711,6 +1711,12 @@ def cmp(df, cmp_fn, value, **kwargs):
 
         run_and_compare(cmp, data=data, cmp_fn=cmp_fn, value=value)
 
+    def test_filter_dtypes(self):
+        def filter(df, **kwargs):
+            return df[df.a < 4].dtypes
+
+        run_and_compare(filter, data=self.cmp_data)
+
     @pytest.mark.xfail(
         reason="Requires fix in OmniSci: https://github.com/intel-ai/omniscidb/pull/178"
     )
Repository: modin-project/modin

Issue:
Keyword arguments 'private_key' and 'verbose' are not supported parameter names of function pandas.read_gbq.
https://github.com/modin-project/modin/blob/02a34eb8c89f1966e41be9a5742ce09a7a6628b2/modin/engines/base/io/io.py#L348


Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/io.py b/modin/engines/base/io/io.py
--- a/modin/engines/base/io/io.py
+++ b/modin/engines/base/io/io.py
@@ -357,8 +357,6 @@ def read_gbq(
                 configuration=configuration,
                 credentials=credentials,
                 use_bqstorage_api=use_bqstorage_api,
-                private_key=private_key,
-                verbose=verbose,
                 progress_bar_type=progress_bar_type,
                 max_results=max_results,
             )


Test patch:
Repository: modin-project/modin

Issue:
Update pandas to 1.3.3



Discussion:

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -13,7 +13,7 @@
 
 import pandas
 
-__pandas_version__ = "1.3.2"
+__pandas_version__ = "1.3.3"
 
 if pandas.__version__ != __pandas_version__:
     import warnings
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -60,7 +60,7 @@ def is_pure(self):
     url="https://github.com/modin-project/modin",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=["pandas==1.3.2", "packaging", "numpy>=1.16.5"],
+    install_requires=["pandas==1.3.3", "packaging", "numpy>=1.16.5"],
     extras_require={
         # can be installed by pip install modin[dask]
         "dask": dask_deps,


Test patch:
Repository: modin-project/modin

Issue:
Unify `PyDbEngine` import
There are two approaches of `PyDbEngine` import now - [the first](https://github.com/modin-project/modin/commit/ac8e2d18826ebe7666bfd3c48fd0489b6b0426ea#diff-dab3aaacc5271b6af37e45c12fd5755e207c9150d09715d731e809493564cc92R111-R117) and [the second](https://github.com/modin-project/modin/blob/master/modin/experimental/engines/omnisci_on_native/frame/omnisci_worker.py#L27-L30). These approaches should be unified to avoid different undesired effects during `PyDbEngine` import.


Discussion:

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -109,10 +109,10 @@ def _get_default(cls):
                 )
             return "Dask"
         try:
-            import omniscidbe  # noqa
-        except ImportError:
+            from omniscidbe import PyDbEngine  # noqa
+        except ModuleNotFoundError:
             try:
-                import dbe  # noqa
+                from dbe import PyDbEngine  # noqa
             except ImportError:
                 pass
             else:


Test patch:
Repository: modin-project/modin

Issue:
AttributeError: module 'modin.pandas' has no attribute 'api'
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Linux Ubuntu 20.04
- **Modin version** (`modin.__version__`): 0.10.2
- **Python version**: 3.8.10
- **Code we can use to reproduce**: 

import modin.pandas as pd

indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=d)
df['new'] = df['rough'].rolling(window=indexer).apply(np.prod, raw=True) - 1


### Describe the problem
modin should have defaulted to pandas in case this functionality is not implemented instead of throwing an error.

### Source code / logs
    indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=d)
AttributeError: module 'modin.pandas' has no attribute 'api'


Discussion:
Thanks @soumava-das for the report!

@prutskov Are you able to take a look? It might just need to be rolled into the default api.
@devin-petersohn Yes, I'll take a look on this.

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -85,6 +85,7 @@
     datetime,
     NamedAgg,
     NA,
+    api,
 )
 import os
 import multiprocessing
@@ -323,6 +324,7 @@ def init_remote_ray(partition):
     "value_counts",
     "datetime",
     "NamedAgg",
+    "api",
 ]
 
 del pandas, Engine, Parameter


Test patch:
diff --git a/modin/pandas/test/test_rolling.py b/modin/pandas/test/test_rolling.py
--- a/modin/pandas/test/test_rolling.py
+++ b/modin/pandas/test/test_rolling.py
@@ -16,7 +16,7 @@
 import pandas
 import modin.pandas as pd
 
-from .utils import df_equals, test_data_values, test_data_keys
+from .utils import df_equals, test_data_values, test_data_keys, create_test_dfs
 from modin.config import NPartitions
 
 NPartitions.put(4)
@@ -204,3 +204,11 @@ def test_series_dt_index(closed):
     )
     df_equals(modin_rolled.aggregate(np.sum), pandas_rolled.aggregate(np.sum))
     df_equals(modin_rolled.quantile(0.1), pandas_rolled.quantile(0.1))
+
+
+def test_api_indexer():
+    modin_df, pandas_df = create_test_dfs(test_data_values[0])
+    indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=3)
+    pandas_rolled = pandas_df.rolling(window=indexer)
+    modin_rolled = modin_df.rolling(window=indexer)
+    df_equals(modin_rolled.sum(), pandas_rolled.sum())