internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
Add MODIN_ENGINE=native to examples dockerfiles when new Modin release is pushed to conda
Current example Dockerfiles build container images installing Modin from conda. Current Modin release 0.10.2 requires `MODIN_ENGINE=ray MODIN_BACKEND=omnisci MODIN_EXPERIMENTAL=true` to activate omnisci backend and this is how environment variables are currently set in Dockerfiles.
After PR #3352 is merged this will change to `MODIN_ENGINE=native MODIN_BACKEND=omnisci MODIN_EXPERIMENTAL=true`, but changes to Dockerfiles should not be done until new Modin release with PR #3352 is published in conda. Then examples will start building from new package versions that require `MODIN_ENGINE=native` to be set, so this variable will have to be set in Dockerfiles.


Discussion:

Resolving patch:
diff --git a/examples/docker/modin-omnisci/census-omnisci.py b/examples/docker/modin-omnisci/census-omnisci.py
--- a/examples/docker/modin-omnisci/census-omnisci.py
+++ b/examples/docker/modin-omnisci/census-omnisci.py
@@ -14,7 +14,7 @@
 import sys
 import time
 import modin.pandas as pd
-from modin.experimental.engines.omnisci_on_ray.frame.omnisci_worker import OmnisciServer
+from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer
 
 from sklearn import config_context
 import sklearnex
diff --git a/examples/docker/modin-omnisci/nyc-taxi-omnisci.py b/examples/docker/modin-omnisci/nyc-taxi-omnisci.py
--- a/examples/docker/modin-omnisci/nyc-taxi-omnisci.py
+++ b/examples/docker/modin-omnisci/nyc-taxi-omnisci.py
@@ -14,7 +14,7 @@
 import sys
 import time
 import modin.pandas as pd
-from modin.experimental.engines.omnisci_on_ray.frame.omnisci_worker import OmnisciServer
+from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer
 
 
 def read(filename):
diff --git a/examples/docker/modin-omnisci/plasticc-omnisci.py b/examples/docker/modin-omnisci/plasticc-omnisci.py
--- a/examples/docker/modin-omnisci/plasticc-omnisci.py
+++ b/examples/docker/modin-omnisci/plasticc-omnisci.py
@@ -16,7 +16,7 @@
 from collections import OrderedDict
 from functools import partial
 import modin.pandas as pd
-from modin.experimental.engines.omnisci_on_ray.frame.omnisci_worker import OmnisciServer
+from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer
 
 import numpy as np
 import xgboost as xgb


Test patch:
Repository: modin-project/modin

Issue:
Add merge benchmark on omnisci where how="inner" and the key is unique
Some interesting case for performance tracking.


Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/omnisci/benchmarks.py b/asv_bench/benchmarks/omnisci/benchmarks.py
--- a/asv_bench/benchmarks/omnisci/benchmarks.py
+++ b/asv_bench/benchmarks/omnisci/benchmarks.py
@@ -51,24 +51,33 @@ class TimeMerge:
     param_names = ["shapes", "how"]
     params = [
         get_benchmark_shapes("omnisci.TimeMerge"),
-        ["left"],
+        ["left", "inner"],
     ]
 
     def setup(self, shapes, how):
-        self.df1 = generate_dataframe(
-            ASV_USE_IMPL, "int", *shapes[0], RAND_LOW, RAND_HIGH
-        )
-        self.df2 = generate_dataframe(
-            ASV_USE_IMPL, "int", *shapes[1], RAND_LOW, RAND_HIGH
-        )
-        trigger_import(self.df1, self.df2)
+        gen_unique_key = how == "inner"
+        self.dfs = []
+        for shape in shapes:
+            self.dfs.append(
+                generate_dataframe(
+                    ASV_USE_IMPL,
+                    "int",
+                    *shape,
+                    RAND_LOW,
+                    RAND_HIGH,
+                    gen_unique_key=gen_unique_key,
+                )
+            )
+        trigger_import(*self.dfs)
 
     def time_merge(self, shapes, how):
         # merging dataframes by index is not supported, therefore we merge by column
         # with arbitrary values, which leads to an unpredictable form of the operation result;
         # it's need to get the predictable shape to get consistent performance results
         execute(
-            self.df1.merge(self.df2, on="col1", how=how, suffixes=("left_", "right_"))
+            self.dfs[0].merge(
+                self.dfs[1], on="col1", how=how, suffixes=("left_", "right_")
+            )
         )
 
 
diff --git a/asv_bench/benchmarks/utils/common.py b/asv_bench/benchmarks/utils/common.py
--- a/asv_bench/benchmarks/utils/common.py
+++ b/asv_bench/benchmarks/utils/common.py
@@ -308,6 +308,7 @@ def generate_dataframe(
     rand_high: int,
     groupby_ncols: Optional[int] = None,
     count_groups: Optional[int] = None,
+    gen_unique_key: bool = False,
 ) -> Union[pd.DataFrame, pandas.DataFrame]:
     """
     Generate DataFrame with caching.
@@ -338,6 +339,8 @@ def generate_dataframe(
         in each group every benchmarking time.
     count_groups : int, default: None
         Count of groups in groupby columns.
+    gen_unique_key : bool, default: False
+        Generate `col1` column where all elements are unique.
 
     Returns
     -------
@@ -363,6 +366,7 @@ def generate_dataframe(
         rand_high,
         groupby_ncols,
         count_groups,
+        gen_unique_key,
     )
 
     if cache_key in dataframes_cache:
@@ -380,6 +384,9 @@ def generate_dataframe(
         for groupby_col in groupby_columns:
             data[groupby_col] = np.tile(np.arange(count_groups), nrows // count_groups)
 
+    if gen_unique_key:
+        data["col1"] = np.arange(nrows)
+
     if impl == "modin":
         df = pd.DataFrame(data)
     elif impl == "pandas":


Test patch:
Repository: modin-project/modin

Issue:
Unpin `boto3` in setup.py
`boto3` pinned only in setup.py. Because of `boto3` pinned user can't install `s3fs` in modin environment due to unsolvable dependencies.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,7 @@ def is_pure(self):
 
 dask_deps = ["dask>=2.22.0", "distributed>=2.22.0"]
 ray_deps = ["ray[default]>=1.4.0", "pyarrow>=1.0"]
-remote_deps = ["rpyc==4.1.5", "cloudpickle", "boto3==1.4.8"]
+remote_deps = ["rpyc==4.1.5", "cloudpickle", "boto3"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]
 sql_deps = ["dfsql>=0.4.2"]
 all_deps = dask_deps + ray_deps + remote_deps + spreadsheet_deps


Test patch:
Repository: modin-project/modin

Issue:
FutureWarning: pandas.core.index is deprecated and will be removed in a future version.  The public classes are available in the top-level namespace.
We need to rewrite: https://github.com/modin-project/modin/blob/f284bb3cfd6e53b468da09efd6edc32bafd10689/modin/experimental/engines/omnisci_on_native/frame/data.py#L20


Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_native/frame/data.py b/modin/experimental/engines/omnisci_on_native/frame/data.py
--- a/modin/experimental/engines/omnisci_on_native/frame/data.py
+++ b/modin/experimental/engines/omnisci_on_native/frame/data.py
@@ -17,7 +17,7 @@
 from modin.experimental.backends.omnisci.query_compiler import DFAlgQueryCompiler
 from .partition_manager import OmnisciOnNativeFramePartitionManager
 
-from pandas.core.index import ensure_index, Index, MultiIndex, RangeIndex
+from pandas.core.indexes.api import ensure_index, Index, MultiIndex, RangeIndex
 from pandas.core.dtypes.common import get_dtype, is_list_like, is_bool_dtype
 from modin.error_message import ErrorMessage
 from modin.pandas.indexing import is_range_like


Test patch:
Repository: modin-project/modin

Issue:
`fsspec` should be explicitly stated in setup.py and env files
`fsspec` package became required dependency after https://github.com/modin-project/modin/pull/3529


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -60,7 +60,7 @@ def is_pure(self):
     url="https://github.com/modin-project/modin",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=["pandas==1.3.3", "packaging", "numpy>=1.16.5"],
+    install_requires=["pandas==1.3.3", "packaging", "numpy>=1.16.5", "fsspec"],
     extras_require={
         # can be installed by pip install modin[dask]
         "dask": dask_deps,


Test patch:
Repository: modin-project/modin

Issue:
FileNotFoundError Failed to open local file
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Manjaro Linux 21.1.0
- **Modin version** (`modin.__version__`): 0.10.1
- **Python version**: 3.9.6
- **Code we can use to reproduce**:
```
import os
os.environ["MODIN_ENGINE"] = "ray"
import pandas
import sys
import modin.pandas as mpd
import time
import ray

ray.init()

os.chdir("theDirectoryWhereTheFileIs")
fileFeatherModin = mpd.read_feather("testFile.feather")
```
### Describe the problem
If I try to read a feather file with Modin, the file isn't found. But it's found by the classical read_feather() function.

### Source code / logs

https://pastebin.com/Xfj2dbRg

EDIT : I tried with read_csv (not the same file obviously), and even if Modin was slower than Pandas, it read the file.

EDIT (07/24/2021) : I found that the problem comes from the change of directory through os.chdir(). Now there's 2 workarounds, coming from **https://github.com/modin-project/modin/issues/882**. Either we put `os.environ["MODIN_ENGINE"] = "ray" ; import modin.pandas as mpd ; import ray and ray.init()` after the os.chdir(), either we put the absolute path of the file we wanna read. The problem seems to occur only with read_feather().

I tested it and only ray.init() needs to be after the os.chdir().
The two workarounds works like this:
```
import os
os.environ["MODIN_ENGINE"] = "ray"
import pandas
import sys
import modin.pandas as mpd
import time
import ray

os.chdir("aDirectory")
ray.init()
fileFeatherModin = mpd.read_feather("testFile.feather")
```

OR

```
import os
os.environ["MODIN_ENGINE"] = "ray"
import pandas
import sys
import modin.pandas as mpd
import time
import ray

ray.init()

fileFeatherModin = mpd.read_feather("aDirectory/testFile.feather")
```


Discussion:
Thanks for the detailed fix @romainRoch!

We can fix this by getting the absolute path of the file for feather before we hand off the work to the workers. `os.chdir` causes problems with Ray / Dask workers (as does `sys.path` in #3362), and this was the light weight fix we used for `read_csv`, etc.

Resolving patch:
diff --git a/modin/core/io/column_stores/feather_dispatcher.py b/modin/core/io/column_stores/feather_dispatcher.py
--- a/modin/core/io/column_stores/feather_dispatcher.py
+++ b/modin/core/io/column_stores/feather_dispatcher.py
@@ -52,6 +52,7 @@ def _read(cls, path, columns=None, **kwargs):
         PyArrow feather is used. Please refer to the documentation here
         https://arrow.apache.org/docs/python/api.html#feather-format
         """
+        path = cls.get_path(path)
         if columns is None:
             from pyarrow.feather import read_feather
 


Test patch:
Repository: modin-project/modin

Issue:
ModuleNotFoundError: No module named 'botocore' when using OpenFile
Need to rewrite this so that this dependency is imported only when the S3 store is being used.
Thereby making it optional like s3fs.


Discussion:

Resolving patch:
diff --git a/modin/core/io/file_dispatcher.py b/modin/core/io/file_dispatcher.py
--- a/modin/core/io/file_dispatcher.py
+++ b/modin/core/io/file_dispatcher.py
@@ -72,18 +72,21 @@ def __enter__(self):
         fsspec.core.OpenFile
             The opened file.
         """
-        from botocore.exceptions import NoCredentialsError
+        try:
+            from botocore.exceptions import NoCredentialsError
+
+            credential_error_type = (NoCredentialsError,)
+        except ModuleNotFoundError:
+            credential_error_type = ()
+
+        args = (self.file_path, self.mode, self.compression)
 
+        self.file = fsspec.open(*args, anon=False)
         try:
-            self.file = fsspec.open(
-                self.file_path, self.mode, self.compression, anon=False
-            )
-            return self.file.open()
-        except NoCredentialsError:
-            self.file = fsspec.open(
-                self.file_path, self.mode, self.compression, anon=True
-            )
             return self.file.open()
+        except credential_error_type:
+            self.file = fsspec.open(*args, anon=True)
+        return self.file.open()
 
     def __exit__(self, *args):
         """


Test patch:
Repository: modin-project/modin

Issue:
Update pandas to 1.3.4



Discussion:

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -13,7 +13,7 @@
 
 import pandas
 
-__pandas_version__ = "1.3.3"
+__pandas_version__ = "1.3.4"
 
 if pandas.__version__ != __pandas_version__:
     import warnings
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -60,7 +60,7 @@ def is_pure(self):
     url="https://github.com/modin-project/modin",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=["pandas==1.3.3", "packaging", "numpy>=1.16.5", "fsspec"],
+    install_requires=["pandas==1.3.4", "packaging", "numpy>=1.16.5", "fsspec"],
     extras_require={
         # can be installed by pip install modin[dask]
         "dask": dask_deps,


Test patch:
Repository: modin-project/modin

Issue:
`read_fwf` does not play nice with `gzip`
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 20.04.3 LTS
- **Modin version** (`modin.__version__`): 0.11.0
- **Python version**: 3.9.7
- **Code we can use to reproduce**:

Download one of the `state*.gz` files from [https://cdiac.ess-dive.lbl.gov/ftp/ushcn_daily/](https://cdiac.ess-dive.lbl.gov/ftp/ushcn_daily/).

```python
from modin import pandas as pd
import gzip

with gzip.open("state01_AL.txt.gz") as file:
    df = pd.read_fwf(file)
```

### Describe the problem
<!-- Describe the problem clearly here. -->

Using the file-object returned from `gzip` causes fallback to regular `pandas`

The related issue for `read_csv`: https://github.com/modin-project/modin/issues/630 was fixed in https://github.com/modin-project/modin/pull/682.



Discussion:
Hi @randolf-scholz, thanks for posting!

This should be relatively straightforward to add, we'll look into it for the next release. Thanks again for the report!
if `read_csv` works well in the case maybe https://github.com/modin-project/modin/pull/3540 can help with the issue.

UPD: `read_csv` also default to pandas in case. So #3540 doesn't help. To fix this we should start from: https://github.com/modin-project/modin/blob/01a5de21c8fb3f6da0f9f887f6a659b7037136a0/modin/core/io/text/text_file_dispatcher.py#L66

gzip reader has another type.
Was this a regression @anmyachev ? Parallel gzip was supported.
> Was this a regression @anmyachev ? Parallel gzip was supported.

Parallel gzip supported only if `filepath_or_buffer` of `str` type, gzip' file object isn't supported.

Resolving patch:
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -23,7 +23,6 @@
 from modin.utils import _inherit_docstrings
 import numpy as np
 import warnings
-import io
 import os
 from typing import Union, Sequence, Optional, Tuple
 import pandas
@@ -63,7 +62,7 @@ def get_path_or_buffer(cls, filepath_or_buffer):
         use it without having to fall back to pandas and share file objects between
         workers. Given a filepath, return it immediately.
         """
-        if isinstance(filepath_or_buffer, (io.BufferedReader, io.TextIOWrapper)):
+        if hasattr(filepath_or_buffer, "name"):
             buffer_filepath = filepath_or_buffer.name
             if cls.file_exists(buffer_filepath):
                 warnings.warn(


Test patch:
Repository: modin-project/modin

Issue:
Pin `pyparsing` to make `dfsql` work
`dfsql` doesn't work with new versions of `pyparsing` (since 3.0.0). We should probably pin `pyparsing` to make `dfsql` work until `dfsql` fixes that on its side.


Discussion:
https://github.com/mindsdb/dfsql/issues/41

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ def is_pure(self):
 ray_deps = ["ray[default]>=1.4.0", "pyarrow>=1.0"]
 remote_deps = ["rpyc==4.1.5", "cloudpickle", "boto3"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]
-sql_deps = ["dfsql>=0.4.2"]
+sql_deps = ["dfsql>=0.4.2", "pyparsing<=2.4.7"]
 all_deps = dask_deps + ray_deps + remote_deps + spreadsheet_deps
 
 # dfsql does not support Windows yet


Test patch:
Repository: modin-project/modin

Issue:
Fix typos in the `modin.core.io.text.csv_dispatcher`
There are a couple of typos in the `modin.core.io.text.csv_dispatcher` in the function docstring, they should be fixed.


Discussion:

Resolving patch:
diff --git a/modin/core/io/text/csv_dispatcher.py b/modin/core/io/text/csv_dispatcher.py
--- a/modin/core/io/text/csv_dispatcher.py
+++ b/modin/core/io/text/csv_dispatcher.py
@@ -420,11 +420,11 @@ def _manage_skiprows_parameter(
         to `skiprows`=[1,2,3]. Now, to avoid this discrepancy, we need to assign
         the first partition to read data between header line and the first
         row for skipping by setting of `pre_reading` parameter, so setting
-        `pre_reading`=2. During data file partitiong, these lines will be assigned
+        `pre_reading`=2. During data file partitioning, these lines will be assigned
         for reading for the first partition, and then file position will be set at
         the beginning of rows that should be skipped by `skiprows_partitioning`.
         After skipping of these rows, the rest data will be divided between the
-        rest of partitions, see rows assignement below:
+        rest of partitions, see rows assignment below:
 
         0 - header line (skip during partitioning)
         1 - pre_reading (assign to read by the first partition)


Test patch:
Repository: modin-project/modin

Issue:
OmniSci GroupBy tests fail with OmniSci 5.8.0
```
FAILED modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py::TestGroupby::test_series_astype
FAILED modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py::TestGroupby::test_df_astype
FAILED modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py::TestGroupby::test_taxi_q4[True]
FAILED modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py::TestGroupby::test_taxi_q4[False]
FAILED modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py::TestGroupby::test_taxi_q4[None]
```

@dchigarev , take a look, please.


Discussion:
Looks like something wrong with type inference and value cast.

```python
from modin.config import IsExperimental, Engine, Backend

IsExperimental.put(True)
Engine.put("native")
Backend.put("omnisci")

import modin.pandas as pd
import pandas
pdf = pandas.DataFrame([1.2, 2.9])
mdf = pd.DataFrame([1.2, 2.9])
print(pdf.astype("int"))
   0
0  1
1  2
print(mdf.astype("int"))
   0
0  1
1  3
```
Oh, it is known problem - we need to pin to 5.7.1 omnisci version so far
When do they plan to fix it?
OmniSci has [fixed casting rule](https://github.com/omnisci/omniscidb/commit/32674087a87cf8582c30b3fd88a2e959d5c9aeff) for `INT` in 5.8.0, now it follows SQL convention (round instead of truncate):
```python
# Omnisci 5.8.0:
>>> server.select_df("select * from test_table")
     a
0  1.2
1  1.5
2  1.6
>>> server.select_df("select CAST(a as INT) as a from test_table")
   a
0  1
1  2
2  2

# Omnisci 5.7.0:
>>> server.select_df("select * from test_table")
     a
0  1.2
1  1.5
2  1.6
>>> server.select_df("select CAST(a as INT) as a from test_table")
   a
0  1
1  1
2  1
```
As a proper fix, we probably need to change SQL-gen for `.astype("int")` from `CAST(col as INT)` to `CAST(FLOOR(col) as INT)`
There is #3538 which is responsible for support Omnisci 5.8

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py
--- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py
+++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py
@@ -219,6 +219,11 @@ def cast(self, res_type):
         BaseExpr
             The cast expression.
         """
+        # From float to int cast we expect truncate behavior but CAST
+        # operation would give us round behavior.
+        if is_float_dtype(self._dtype) and is_integer_dtype(res_type):
+            return self.floor()
+
         new_expr = OpExpr("CAST", [self], res_type)
         return new_expr
 


Test patch:
Repository: modin-project/modin

Issue:
`set_index` with heterogeneous objects defaults to pandas
We support `set_index` with List/Index objects, and with column names, but not a combination of the two. For example:

```python
df.set_index([df.index, "col0"])
```


Discussion:

Resolving patch:
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -1955,24 +1955,17 @@ def set_index(
             isinstance(col, (pandas.Index, Series, np.ndarray, list, Iterator))
             for col in keys
         ):
-            # The current implementation cannot mix a list column labels and list like
-            # objects.
-            if not all(
-                isinstance(col, (pandas.Index, Series, np.ndarray, list, Iterator))
-                for col in keys
-            ):
-                return self._default_to_pandas(
-                    "set_index",
-                    keys,
-                    drop=drop,
-                    append=append,
-                    inplace=inplace,
-                    verify_integrity=verify_integrity,
-                )
             if inplace:
                 frame = self
             else:
                 frame = self.copy()
+            if not all(
+                isinstance(col, (pandas.Index, Series, np.ndarray, list, Iterator))
+                for col in keys
+            ):
+                if drop:
+                    keys = [frame.pop(k) if not is_list_like(k) else k for k in keys]
+                keys = [k._to_pandas() if isinstance(k, Series) else k for k in keys]
             # These are single-threaded objects, so we might as well let pandas do the
             # calculation so that it matches.
             frame.index = (


Test patch:
diff --git a/modin/pandas/test/dataframe/test_indexing.py b/modin/pandas/test/dataframe/test_indexing.py
--- a/modin/pandas/test/dataframe/test_indexing.py
+++ b/modin/pandas/test/dataframe/test_indexing.py
@@ -267,6 +267,16 @@ def test_indexing_duplicate_axis(data):
     )
 
 
+@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+def test_set_index(data):
+    modin_df = pd.DataFrame(data)
+    pandas_df = pandas.DataFrame(data)
+
+    modin_result = modin_df.set_index([modin_df.index, modin_df.columns[0]])
+    pandas_result = pandas_df.set_index([pandas_df.index, pandas_df.columns[0]])
+    df_equals(modin_result, pandas_result)
+
+
 @pytest.mark.gpu
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_keys(data):