internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
REFACTOR: __invert__ should raise a TypeError at API layer for non-integer types
We raise the correct TypeError, but we do at the execution layer.


Discussion:
API layer has check for numeric dtype but should check for integer dtype: https://github.com/modin-project/modin/blob/781fed8152a6e0a6fcae9cab5604644577103a5f/modin/pandas/base.py#L3696

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -20,6 +20,8 @@
 from pandas.core.dtypes.common import (
     is_list_like,
     is_dict_like,
+    is_bool_dtype,
+    is_integer_dtype,
     is_numeric_dtype,
     is_datetime_or_timedelta_dtype,
     is_dtype_equal,
@@ -3693,10 +3695,14 @@ def __invert__(self):
         BasePandasDataset
             New BasePandasDataset containing bitwise inverse to each value.
         """
-        if not all(is_numeric_dtype(d) for d in self._get_dtypes()):
+        if not all(is_bool_dtype(d) or is_integer_dtype(d) for d in self._get_dtypes()):
             raise TypeError(
                 "bad operand type for unary ~: '{}'".format(
-                    next(d for d in self._get_dtypes() if not is_numeric_dtype(d))
+                    next(
+                        d
+                        for d in self._get_dtypes()
+                        if not (is_bool_dtype(d) or is_integer_dtype(d))
+                    )
                 )
             )
         return self.__constructor__(query_compiler=self._query_compiler.invert())


Test patch:
diff --git a/modin/pandas/test/dataframe/test_map_metadata.py b/modin/pandas/test/dataframe/test_map_metadata.py
--- a/modin/pandas/test/dataframe/test_map_metadata.py
+++ b/modin/pandas/test/dataframe/test_map_metadata.py
@@ -1642,6 +1642,15 @@ def test___invert__(data):
         df_equals(modin_result, pandas_result)
 
 
+def test___invert___bool():
+    data = [False]
+    modin_df = pd.DataFrame(data)
+    pandas_df = pandas.DataFrame(data)
+    modin_result = ~modin_df
+    pandas_result = ~pandas_df
+    df_equals(modin_result, pandas_result)
+
+
 def test___hash__():
     data = test_data_values[0]
     pandas_df = pandas.DataFrame(data)
Repository: modin-project/modin

Issue:
FEAT: HDK: Add support for datetime64 to int64 cast
Currently, the following code defaults to pandas:

```python
df = pd.DataFrame({"date": [1, 2, 3]}, dtype="datetime64[ns]")
print(df.astype("int64"))
```


Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py
@@ -25,6 +25,7 @@
     is_list_like,
     get_dtype,
     is_float_dtype,
+    is_int64_dtype,
     is_integer_dtype,
     is_numeric_dtype,
     is_string_dtype,
@@ -69,6 +70,10 @@ def _get_common_dtype(lhs_dtype, rhs_dtype):
         return get_dtype(int)
     if is_datetime64_dtype(lhs_dtype) and is_datetime64_dtype(rhs_dtype):
         return np.promote_types(lhs_dtype, rhs_dtype)
+    if (is_datetime64_dtype(lhs_dtype) and is_int64_dtype(rhs_dtype)) or (
+        is_datetime64_dtype(rhs_dtype) and is_int64_dtype(lhs_dtype)
+    ):
+        return get_dtype(int)
     raise NotImplementedError(
         f"Cannot perform operation on types: {lhs_dtype}, {rhs_dtype}"
     )


Test patch:
Repository: modin-project/modin

Issue:
FEAT: HDK: Add 'enable_multifrag_execution_result=1' HDK launch parameter
It could significantly improve performance of queries, executed on results from the previous queries.


Discussion:

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -573,6 +573,7 @@ def _get_default(cls) -> Any:
                 "null_div_by_zero": 1,
                 "enable_watchdog": 0,
                 "enable_thrift_logs": 0,
+                "enable_multifrag_execution_result": 1,
                 "cpu_only": 1,
             }
 


Test patch:
Repository: modin-project/modin

Issue:
HDK: test_dataframe.py is crashed if Calcite is disabled 
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
...
```


### Issue Description

Set the environment variable `MODIN_USE_CALCITE=False` and run the test.

### Expected Behavior

Test passes.

### Error Logs

<details>

```python-traceback

Replace this line with the error backtrace (if applicable).

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py
@@ -146,14 +146,19 @@ def __init__(self, relOp):
         self.relOp = relOp
 
     @classmethod
-    def reset_id(cls):
+    def reset_id(cls, next_id=0):
         """
-        Reset ID to be used for the next new node to 0.
+        Reset ID to be used for the next new node to `next_id`.
 
         Can be used to have a zero-based numbering for each
         generated query.
+
+        Parameters
+        ----------
+        next_id : int, default: 0
+            Next node id.
         """
-        cls._next_id[0] = 0
+        cls._next_id[0] = next_id
 
 
 class CalciteScanNode(CalciteBaseNode):
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py
@@ -754,8 +754,10 @@ def _push(self, node):
             and all(isinstance(expr, CalciteInputRefExpr) for expr in node.exprs)
         ):
             # Replace the last CalciteProjectionNode with this one and
-            # translate the input refs.
-            exprs = self.res.pop().exprs
+            # translate the input refs. The `id` attribute is preserved.
+            last = self.res.pop()
+            exprs = last.exprs
+            last.reset_id(int(last.id))
             node = CalciteProjectionNode(
                 node.fields, [exprs[expr.input] for expr in node.exprs]
             )


Test patch:
Repository: modin-project/modin

Issue:
`mean` doesn't compute dtypes
The operation is implemented through the `TreeReduce` operator. It may be possible to implement the `infer_dtypes` parameter, as it is done for the `Binary` operator.


Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -972,9 +972,25 @@ def reduce_fn(df, **kwargs):
                 count_cols = count_cols.sum(axis=axis, skipna=False)
             return sum_cols / count_cols
 
+        def compute_dtypes_fn(dtypes, axis, **kwargs):
+            """
+            Compute the resulting Series dtype.
+
+            When computing along rows and there are numeric and boolean columns
+            Pandas returns `object`. In all other cases - `float64`.
+            """
+            if (
+                axis == 1
+                and any(is_bool_dtype(t) for t in dtypes)
+                and any(is_numeric_dtype(t) for t in dtypes)
+            ):
+                return "object"
+            return "float64"
+
         return TreeReduce.register(
             map_fn,
             reduce_fn,
+            compute_dtypes=compute_dtypes_fn,
         )(self, axis=axis, **kwargs)
 
     # END TreeReduce operations


Test patch:
Repository: modin-project/modin

Issue:
Add `shape_hint='column'` for the `qc.to_datetime()` result
We seem to only return a one-column qc there, so there should be a type-hint to avoid unnecessary computation triggering at the `.columnarize()` call inside of `pd.Series` constructor
https://github.com/modin-project/modin/blob/0fa3dda01a25a84fc7546424f5273532ac37c3f8/modin/core/storage_formats/pandas/query_compiler.py#L1026


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/algebra/reduce.py b/modin/core/dataframe/algebra/reduce.py
--- a/modin/core/dataframe/algebra/reduce.py
+++ b/modin/core/dataframe/algebra/reduce.py
@@ -20,7 +20,7 @@ class Reduce(Operator):
     """Builder class for Reduce operator."""
 
     @classmethod
-    def register(cls, reduce_function, axis=None):
+    def register(cls, reduce_function, axis=None, shape_hint=None):
         """
         Build Reduce operator that will be performed across rows/columns.
 
@@ -32,6 +32,8 @@ def register(cls, reduce_function, axis=None):
             Source function.
         axis : int, optional
             Axis to apply function along.
+        shape_hint : {"row", "column", None}, default: None
+            Shape hint for the results known to be a column or a row, otherwise None.
 
         Returns
         -------
@@ -46,7 +48,8 @@ def caller(query_compiler, *args, **kwargs):
                 query_compiler._modin_frame.reduce(
                     cls.validate_axis(_axis),
                     lambda x: reduce_function(x, *args, **kwargs),
-                )
+                ),
+                shape_hint=shape_hint,
             )
 
         return caller
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -1029,10 +1029,13 @@ def to_datetime(self, *args, **kwargs):
                 # to_datetime has inplace side effects, see GH#3063
                 lambda df, *args, **kwargs: pandas.to_datetime(
                     df.squeeze(axis=1), *args, **kwargs
-                ).to_frame()
+                ).to_frame(),
+                shape_hint="column",
             )(self, *args, **kwargs)
         else:
-            return Reduce.register(pandas.to_datetime, axis=1)(self, *args, **kwargs)
+            return Reduce.register(pandas.to_datetime, axis=1, shape_hint="column")(
+                self, *args, **kwargs
+            )
 
     # END Reduce operations
 


Test patch:
Repository: modin-project/modin

Issue:
Interchange `Column.null_count` is a NumPy scalar, not a builtin `int`
A `PandasProtocolColumn` returns a `null_count` as a 0d integer array (specifically a NumPy scalar), as opposed to [`int` as specified in the interchange protocol](https://github.com/data-apis/dataframe-api/blob/4f7c1e0c425643d57120c5b73434d992b3e83595/protocol/dataframe_protocol.py#L216).

```python
>>> from modin import pandas as mpd
>>> df = mpd.DataFrame({"foo": [42]})
>>> interchange_df = df.__dataframe__()
>>> interchange_col = interchange_df.get_column_by_name("foo")
>>> interchange_col.null_count
0
>>> type(interchange_col.null_count)
numpy.int64  # should be Python's int
```

This seems to be because the `null_count` implementation uses `DataFrame.squeeze()`, which returns a NumPy scalar rather than an `int`.

https://github.com/modin-project/modin/blob/9b33451648a3192e93c46ac6961627ed2858c7fd/modin/core/dataframe/pandas/exchange/dataframe_protocol/column.py#L222-L245

Related https://github.com/pandas-dev/pandas/issues/47789


Discussion:
@honno thank you for reporting the issue. I can reproduce it with your code at 5af9832d7fad3d17f05d63908bc377e61542d953.

Resolving patch:
diff --git a/modin/core/dataframe/pandas/interchange/dataframe_protocol/column.py b/modin/core/dataframe/pandas/interchange/dataframe_protocol/column.py
--- a/modin/core/dataframe/pandas/interchange/dataframe_protocol/column.py
+++ b/modin/core/dataframe/pandas/interchange/dataframe_protocol/column.py
@@ -251,7 +251,7 @@ def reduce_func(df):
         # Otherwise, we get mismatching internal and external indices for both axes
         intermediate_df.index = pandas.RangeIndex(1)
         intermediate_df.columns = pandas.RangeIndex(1)
-        self._null_count_cache = intermediate_df.to_pandas().squeeze()
+        self._null_count_cache = intermediate_df.to_pandas().squeeze(axis=1).item()
         return self._null_count_cache
 
     @property


Test patch:
diff --git a/modin/test/interchange/dataframe_protocol/test_general.py b/modin/test/interchange/dataframe_protocol/test_general.py
--- a/modin/test/interchange/dataframe_protocol/test_general.py
+++ b/modin/test/interchange/dataframe_protocol/test_general.py
@@ -82,6 +82,14 @@ def test_na_float(df_from_dict):
     assert colX.null_count == 1
 
 
+def test_null_count(df_from_dict):
+    df = df_from_dict({"foo": [42]})
+    dfX = df.__dataframe__()
+    colX = dfX.get_column_by_name("foo")
+    null_count = colX.null_count
+    assert null_count == 0 and type(null_count) is int
+
+
 def test_noncategorical(df_from_dict):
     df = df_from_dict({"a": [1, 2, 3]})
     dfX = df.__dataframe__()
Repository: modin-project/modin

Issue:
Make `pd.read_csv_glob(s3_dir)` work with `s3fs>=2023.9.0`
This is a follow-up issue to investigate why our tests started to fail after the new s3fs release (#6535).

The s3fs version was pinned by #6536, but we still need to investigate and fix the root cause and remove the pin


Discussion:
https://github.com/fsspec/s3fs/issues/783

Resolving patch:
diff --git a/modin/experimental/core/io/text/csv_glob_dispatcher.py b/modin/experimental/core/io/text/csv_glob_dispatcher.py
--- a/modin/experimental/core/io/text/csv_glob_dispatcher.py
+++ b/modin/experimental/core/io/text/csv_glob_dispatcher.py
@@ -350,7 +350,14 @@ def get_path(cls, file_path: str) -> list:
             credential_error_type = (PermissionError,)
 
         def get_file_path(fs_handle) -> List[str]:
-            file_paths = fs_handle.glob(file_path)
+            if "*" in file_path:
+                file_paths = fs_handle.glob(file_path)
+            else:
+                file_paths = [
+                    f
+                    for f in fs_handle.find(file_path)
+                    if not f.endswith("/")  # exclude folder
+                ]
             if len(file_paths) == 0 and not fs_handle.exists(file_path):
                 raise FileNotFoundError(f"Path <{file_path}> isn't available.")
             fs_addresses = [fs_handle.unstrip_protocol(path) for path in file_paths]


Test patch:
Repository: modin-project/modin

Issue:
OmniSci `time_groupby_agg_nunique` benchmark failed in CI
Error: `OmniSci: ERR_OUT_OF_CPU_MEM: Not enough host memory to execute the query`

Initial found: https://github.com/modin-project/modin/pull/4231/files#r811069201

Benchmark sources: https://github.com/modin-project/modin/blob/c17dde71ae8114723a13279e36bf5532dcba328a/asv_bench/benchmarks/omnisci/benchmarks.py#L495

cc @modin-project/modin-omnisci 


Discussion:
Temporary disabled in https://github.com/modin-project/modin/pull/4231 (for testing in CI)

Resolving patch:
diff --git a/asv_bench/benchmarks/utils/data_shapes.py b/asv_bench/benchmarks/utils/data_shapes.py
--- a/asv_bench/benchmarks/utils/data_shapes.py
+++ b/asv_bench/benchmarks/utils/data_shapes.py
@@ -19,7 +19,12 @@
 from .compatibility import ASV_DATASET_SIZE, ASV_USE_STORAGE_FORMAT
 
 RAND_LOW = 0
-RAND_HIGH = 1_000_000_000 if ASV_USE_STORAGE_FORMAT == "hdk" else 100
+# use a small number of unique values in Github actions to avoid OOM (mostly related to HDK)
+RAND_HIGH = (
+    1_000_000_000
+    if ASV_USE_STORAGE_FORMAT == "hdk" and ASV_DATASET_SIZE == "Big"
+    else 100
+)
 
 BINARY_OP_DATA_SIZE = {
     "big": [


Test patch:
Repository: modin-project/modin

Issue:
Don't use deprecated `is_int64_dtype` and `is_period_dtype` functions



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py
@@ -26,7 +26,6 @@
     is_datetime64_any_dtype,
     is_datetime64_dtype,
     is_float_dtype,
-    is_int64_dtype,
     is_integer_dtype,
     is_list_like,
     is_numeric_dtype,
@@ -69,8 +68,8 @@ def _get_common_dtype(lhs_dtype, rhs_dtype):
         return _get_dtype(int)
     if is_datetime64_dtype(lhs_dtype) and is_datetime64_dtype(rhs_dtype):
         return np.promote_types(lhs_dtype, rhs_dtype)
-    if (is_datetime64_dtype(lhs_dtype) and is_int64_dtype(rhs_dtype)) or (
-        is_datetime64_dtype(rhs_dtype) and is_int64_dtype(lhs_dtype)
+    if (is_datetime64_dtype(lhs_dtype) and rhs_dtype == np.int64) or (
+        is_datetime64_dtype(rhs_dtype) and (lhs_dtype == np.int64)
     ):
         return _get_dtype(int)
     raise NotImplementedError(


Test patch:
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -32,7 +32,6 @@
     is_list_like,
     is_numeric_dtype,
     is_object_dtype,
-    is_period_dtype,
     is_string_dtype,
     is_timedelta64_dtype,
 )
@@ -650,7 +649,7 @@ def assert_dtypes_equal(df1, df2):
         lambda obj: isinstance(obj, pandas.CategoricalDtype),
         is_datetime64_any_dtype,
         is_timedelta64_dtype,
-        is_period_dtype,
+        lambda obj: isinstance(obj, pandas.PeriodDtype),
     )
 
     for col in dtypes1.keys():
Repository: modin-project/modin

Issue:
HDK: Use MODIN_CPUS instead of os.cpu_count() for the fragment size calculation
When the HDK engine is initialized, it sets the environment variable OMP_NUM_THREADS = CpuCount.get(). By default this value is equal to the number of CPUs, but it could be changed by the user.


Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py
@@ -12,7 +12,6 @@
 # governing permissions and limitations under the License.
 
 """Module provides ``HdkWorker`` class."""
-import os
 from typing import List, Optional, Tuple, Union
 
 import pyarrow as pa
@@ -20,7 +19,7 @@
 from packaging import version
 from pyhdk.hdk import HDK, ExecutionResult, QueryNode, RelAlgExecutor
 
-from modin.config import HdkFragmentSize, HdkLaunchParameters
+from modin.config import CpuCount, HdkFragmentSize, HdkLaunchParameters
 from modin.utils import _inherit_docstrings
 
 from .base_worker import BaseDbWorker, DbTable
@@ -143,7 +142,7 @@ def compute_fragment_size(cls, table):
         fragment_size = HdkFragmentSize.get()
         if fragment_size is None:
             if cls._preferred_device == "CPU":
-                cpu_count = os.cpu_count()
+                cpu_count = CpuCount.get()
                 if cpu_count is not None:
                     fragment_size = table.num_rows // cpu_count
                     fragment_size = min(fragment_size, 2**25)


Test patch:
Repository: modin-project/modin

Issue:
HDK: Failed to read csv due to error: Unsupported numpy type 17
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
import os
import modin.pandas as pd
file_name = "tmp.csv"
try:
    with open("tmp.csv", "w") as f:
        f.write("test\ntest")
    df = pd.read_csv(file_name, dtype={"test": "object"})
    print(df)
finally:
    os.remove(file_name)
```


### Issue Description

Unable to read string columns if the dtype is specified as `object`.

### Expected Behavior

```
   test
0  test
```

### Error Logs

<details>

```python-traceback

Replace this line with the error backtrace (if applicable).

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
@@ -290,7 +290,7 @@ def _dtype_to_arrow(cls, dtype):
         tname = dtype if isinstance(dtype, str) else dtype.name
         if tname == "category":
             return pa.dictionary(index_type=pa.int32(), value_type=pa.string())
-        elif tname == "string":
+        elif tname == "string" or tname == "object":
             return pa.string()
         else:
             return pa.from_numpy_dtype(tname)


Test patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
@@ -391,6 +391,19 @@ def test(df, lib, **kwargs):
 
         run_and_compare(test, data={})
 
+    def test_read_csv_dtype_object(self):
+        with pytest.warns(UserWarning) as warns:
+            with ensure_clean(".csv") as file:
+                with open(file, "w") as f:
+                    f.write("test\ntest")
+
+                def test(**kwargs):
+                    return pd.read_csv(file, dtype={"test": "object"})
+
+                run_and_compare(test, data={})
+            for warn in warns.list:
+                assert not re.match(r".*defaulting to pandas.*", str(warn))
+
 
 class TestMasks:
     data = {