internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
Upgrade to pandas 0.25.3
Pandas 0.25.3 was released recently: https://pandas.pydata.org/pandas-docs/stable/whatsnew/v0.25.3.html


Discussion:

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -1,6 +1,6 @@
 import pandas
 
-__pandas_version__ = "0.25.2"
+__pandas_version__ = "0.25.3"
 
 if pandas.__version__ != __pandas_version__:
     raise ImportError(
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@
     url="https://github.com/modin-project/modin",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=["pandas==0.25.2"],
+    install_requires=["pandas==0.25.3"],
     extras_require={
         # can be installed by pip install modin[dask]
         "dask": dask_deps,


Test patch:
Repository: modin-project/modin

Issue:
Missing 'title' method in StringMethods object
### System information
- **OS Platform and Distribution: Linux Ubuntu 16.04
- **Modin installed from (source or binary)**: binary
- **Modin version**: 0.6.3
- **Python version**: 3.6.8
- **Exact command to reproduce**:



### Describe the problem
Got AttributeError: 'StringMethods' object has no attribute 'title' when I was trying to execute a line of code contains series.str.title().

I have checked the code in modin/pandas/series.py, method 'title' isn't there. Can you add it in or introduce a workaround? Thanks a lot!

### Source code / logs
modin/pandas/series.py



Discussion:
Hi @jh-wu, thanks for posting!

I will get this fixed before the next release, it shouldn't take long to implement. As a temporary workaround, you can do `series.apply(str.title)`.

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -449,6 +449,7 @@ def transpose(self, *args, **kwargs):
     str_startswith = MapFunction.register(_str_map("startswith"), dtypes=np.bool)
     str_strip = MapFunction.register(_str_map("strip"), dtypes="copy")
     str_swapcase = MapFunction.register(_str_map("swapcase"), dtypes="copy")
+    str_title = MapFunction.register(_str_map("title"), dtypes="copy")
     str_translate = MapFunction.register(_str_map("translate"), dtypes="copy")
     str_upper = MapFunction.register(_str_map("upper"), dtypes="copy")
     str_wrap = MapFunction.register(_str_map("wrap"), dtypes="copy")
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -1523,6 +1523,9 @@ def lower(self):
     def upper(self):
         return Series(query_compiler=self._query_compiler.str_upper())
 
+    def title(self):
+        return Series(query_compiler=self._query_compiler.str_title())
+
     def find(self, sub, start=0, end=None):
         if not isinstance(sub, str):
             raise TypeError(


Test patch:
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -3248,6 +3248,20 @@ def test_str_upper(data):
         df_equals(modin_result, pandas_result)
 
 
+@pytest.mark.parametrize("data", test_string_data_values, ids=test_string_data_keys)
+def test_str_title(data):
+    modin_series, pandas_series = create_test_series(data)
+
+    try:
+        pandas_result = pandas_series.str.title()
+    except Exception as e:
+        with pytest.raises(type(e)):
+            modin_series.str.title()
+    else:
+        modin_result = modin_series.str.title()
+        df_equals(modin_result, pandas_result)
+
+
 @pytest.mark.parametrize("data", test_string_data_values, ids=test_string_data_keys)
 @pytest.mark.parametrize("sub", string_sep_values, ids=string_sep_keys)
 @pytest.mark.parametrize("start", int_arg_values, ids=int_arg_keys)
Repository: modin-project/modin

Issue:
Attribute name in the dataframe
Hi, I am testing modin and I have a problem, but I don't know if this problem is caused by the version of pandas of modin, The attribute "name" in the iloc change between pandas and modin.pandas.

![2](https://user-images.githubusercontent.com/22222488/70291037-f116c500-17a7-11ea-8f5e-ffccc6f80f11.png)




Discussion:
Thanks @DaielChom for reporting!

It looks like the name swapped with the index value between the two during the `iloc` call. I will dig into this and get it fixed for the next release. Thanks again for reporting!

Resolving patch:
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -1532,7 +1532,7 @@ def squeeze(self, axis=None):
         if axis == 1 and len(self.columns) == 1:
             return Series(query_compiler=self._query_compiler)
         if axis == 0 and len(self.index) == 1:
-            return Series(query_compiler=self._query_compiler)
+            return Series(query_compiler=self.T._query_compiler)
         else:
             return self.copy()
 


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -4635,6 +4635,21 @@ def test_squeeze(self):
         ray_df_5 = pd.DataFrame(frame_data_5).squeeze()
         df_equals(ray_df_5, pandas_df_5)
 
+        data = [
+            [
+                pd.Timestamp("2019-01-02"),
+                pd.Timestamp("2019-01-03"),
+                pd.Timestamp("2019-01-04"),
+                pd.Timestamp("2019-01-05"),
+            ],
+            [1, 1, 1, 2],
+        ]
+        df = pd.DataFrame(data, index=["date", "value"]).T
+        pf = pandas.DataFrame(data, index=["date", "value"]).T
+        df.set_index("date", inplace=True)
+        pf.set_index("date", inplace=True)
+        df_equals(df.iloc[0], pf.iloc[0])
+
     def test_stack(self):
         data = test_data_values[0]
         with pytest.warns(UserWarning):
Repository: modin-project/modin

Issue:
Stop uploading Python2 wheels

### Describe the problem
<!-- Describe the problem clearly here. -->
Several Python2 users who install are running into issues, so we should stop uploading wheels for Python2 during the release process.

Related to #880 and #883.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,3 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 from setuptools import setup, find_packages
 
 with open("README.md", "r") as fh:
@@ -25,4 +21,5 @@
         "ray": ray_deps,
         "all": dask_deps + ray_deps,
     },
+    python_requires=">=3.5",
 )


Test patch:
Repository: modin-project/modin

Issue:
ModuleNotFoundError: No module named ‘py’

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem

This was reported on the Discourse: https://discuss.modin.org/t/error-no-module-named-py-in/92
<!-- Describe the problem clearly here. -->

We should update the setup.py to install py.
### Source code / logs

See link above.
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/text/csv_reader.py b/modin/engines/base/io/text/csv_reader.py
--- a/modin/engines/base/io/text/csv_reader.py
+++ b/modin/engines/base/io/text/csv_reader.py
@@ -2,10 +2,27 @@
 from modin.data_management.utils import compute_chunksize
 from pandas.io.parsers import _validate_usecols_arg
 import pandas
-import py
 import sys
 
 
+def pathlib_or_pypath(filepath_or_buffer):
+    try:
+        import py
+
+        if isinstance(filepath_or_buffer, py.path.local):
+            return True
+    except ImportError:  # pragma: no cover
+        pass
+    try:
+        import pathlib
+
+        if isinstance(filepath_or_buffer, pathlib.Path):
+            return True
+    except ImportError:  # pragma: no cover
+        pass
+    return False
+
+
 class CSVReader(TextFileReader):
     @classmethod
     def read(cls, filepath_or_buffer, **kwargs):
@@ -13,18 +30,8 @@ def read(cls, filepath_or_buffer, **kwargs):
             if not cls.file_exists(filepath_or_buffer):
                 return cls.single_worker_read(filepath_or_buffer, **kwargs)
             filepath_or_buffer = cls.get_path(filepath_or_buffer)
-        elif not isinstance(filepath_or_buffer, py.path.local):
-            read_from_pandas = True
-            # Pandas read_csv supports pathlib.Path
-            try:
-                import pathlib
-
-                if isinstance(filepath_or_buffer, pathlib.Path):
-                    read_from_pandas = False
-            except ImportError:  # pragma: no cover
-                pass
-            if read_from_pandas:
-                return cls.single_worker_read(filepath_or_buffer, **kwargs)
+        elif not pathlib_or_pypath(filepath_or_buffer):
+            return cls.single_worker_read(filepath_or_buffer, **kwargs)
         compression_type = cls.infer_compression(
             filepath_or_buffer, kwargs.get("compression")
         )


Test patch:
Repository: modin-project/modin

Issue:
ModuleNotFoundError: No module named ‘py’

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem

This was reported on the Discourse: https://discuss.modin.org/t/error-no-module-named-py-in/92
<!-- Describe the problem clearly here. -->

We should update the setup.py to install py.
### Source code / logs

See link above.
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/text/csv_reader.py b/modin/engines/base/io/text/csv_reader.py
--- a/modin/engines/base/io/text/csv_reader.py
+++ b/modin/engines/base/io/text/csv_reader.py
@@ -2,10 +2,27 @@
 from modin.data_management.utils import compute_chunksize
 from pandas.io.parsers import _validate_usecols_arg
 import pandas
-import py
 import sys
 
 
+def pathlib_or_pypath(filepath_or_buffer):
+    try:
+        import py
+
+        if isinstance(filepath_or_buffer, py.path.local):
+            return True
+    except ImportError:  # pragma: no cover
+        pass
+    try:
+        import pathlib
+
+        if isinstance(filepath_or_buffer, pathlib.Path):
+            return True
+    except ImportError:  # pragma: no cover
+        pass
+    return False
+
+
 class CSVReader(TextFileReader):
     @classmethod
     def read(cls, filepath_or_buffer, **kwargs):
@@ -13,18 +30,8 @@ def read(cls, filepath_or_buffer, **kwargs):
             if not cls.file_exists(filepath_or_buffer):
                 return cls.single_worker_read(filepath_or_buffer, **kwargs)
             filepath_or_buffer = cls.get_path(filepath_or_buffer)
-        elif not isinstance(filepath_or_buffer, py.path.local):
-            read_from_pandas = True
-            # Pandas read_csv supports pathlib.Path
-            try:
-                import pathlib
-
-                if isinstance(filepath_or_buffer, pathlib.Path):
-                    read_from_pandas = False
-            except ImportError:  # pragma: no cover
-                pass
-            if read_from_pandas:
-                return cls.single_worker_read(filepath_or_buffer, **kwargs)
+        elif not pathlib_or_pypath(filepath_or_buffer):
+            return cls.single_worker_read(filepath_or_buffer, **kwargs)
         compression_type = cls.infer_compression(
             filepath_or_buffer, kwargs.get("compression")
         )


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -153,7 +153,7 @@ def make_csv_file(delimiter=",", compression="infer"):
     def _make_csv_file(
         filename=TEST_CSV_FILENAME,
         row_size=SMALL_ROW_SIZE,
-        force=False,
+        force=True,
         delimiter=delimiter,
         encoding=None,
         compression=compression,
Repository: modin-project/modin

Issue:
Skip ray import for Winows and avoid ModuleNotFoundError
Reported originally in #657 



Discussion:

Resolving patch:
diff --git a/modin/__init__.py b/modin/__init__.py
--- a/modin/__init__.py
+++ b/modin/__init__.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import warnings
 
 
@@ -19,39 +20,42 @@ def get_execution_engine():
     # execution engine + backing (Pandas + Ray).
     if "MODIN_ENGINE" in os.environ:
         # .title allows variants like ray, RAY, Ray
-        engine = os.environ["MODIN_ENGINE"].title()
+        return os.environ["MODIN_ENGINE"].title()
     else:
         if "MODIN_DEBUG" in os.environ:
-            engine = "Python"
+            return "Python"
         else:
-            try:
-                import ray
-
-                engine = "Ray"
-            except ImportError:
+            if sys.platform != "win32":
                 try:
-                    import dask
-                    import distributed
+                    import ray
 
-                    engine = "Dask"
                 except ImportError:
-                    raise ImportError(
-                        "Please `pip install modin[ray] or modin[dask] to install an engine"
-                    )
+                    pass
                 else:
-                    if (
-                        str(dask.__version__) < "2.1.0"
-                        or str(distributed.__version__) < "2.3.2"
-                    ):
+                    if ray.__version__ != "0.8.0":
                         raise ImportError(
-                            "Please `pip install modin[dask] to install compatible Dask version."
+                            "Please `pip install modin[ray]` to install compatible Ray version."
                         )
+                    return "Ray"
+            try:
+                import dask
+                import distributed
+
+            except ImportError:
+                raise ImportError(
+                    "Please `pip install {}modin[dask]` to install an engine".format(
+                        "modin[ray]` or `" if sys.platform != "win32" else ""
+                    )
+                )
             else:
-                if ray.__version__ != "0.8.0":
+                if (
+                    str(dask.__version__) < "2.1.0"
+                    or str(distributed.__version__) < "2.3.2"
+                ):
                     raise ImportError(
-                        "Please `pip install modin[ray] to install compatible Ray version."
+                        "Please `pip install modin[dask]` to install compatible Dask version."
                     )
-    return engine
+                return "Dask"
 
 
 def get_partition_format():


Test patch:
Repository: modin-project/modin

Issue:
DeprecationWarning: invalid escape sequence \d
# Source code / logs

Two offending lines:
https://github.com/modin-project/modin/blob/f6b3453a2eeb2100e5ddaa3c328f6254abc96502/modin/engines/ray/generic/io.py#L17

https://github.com/modin-project/modin/blob/272529a9c1f13ed3dde60e524676591b889ff8f7/modin/pandas/io.py#L12

The fix is to prepend string with `r`. E.g. `r"__\d+__"`


Discussion:

Resolving patch:
diff --git a/modin/engines/ray/generic/io.py b/modin/engines/ray/generic/io.py
--- a/modin/engines/ray/generic/io.py
+++ b/modin/engines/ray/generic/io.py
@@ -1,7 +1,6 @@
 import pandas
 
 import os
-import re
 import numpy as np
 import math
 import warnings
@@ -14,8 +13,6 @@
 if __execution_engine__ == "Ray":
     import ray
 
-PQ_INDEX_REGEX = re.compile("__index_level_\d+__")  # noqa W605
-
 
 class RayIO(BaseIO):
 
@@ -88,6 +85,7 @@ def read_parquet(cls, path, engine, columns, **kwargs):
         """
 
         from pyarrow.parquet import ParquetFile, ParquetDataset
+        from modin.pandas.io import PQ_INDEX_REGEX
 
         if cls.read_parquet_remote_task is None:
             return super(RayIO, cls).read_parquet(path, engine, columns, **kwargs)
diff --git a/modin/pandas/io.py b/modin/pandas/io.py
--- a/modin/pandas/io.py
+++ b/modin/pandas/io.py
@@ -9,7 +9,7 @@
 from .dataframe import DataFrame
 from modin.data_management.factories import BaseFactory
 
-PQ_INDEX_REGEX = re.compile("__index_level_\d+__")  # noqa W605
+PQ_INDEX_REGEX = re.compile(r"__index_level_\d+__")
 
 
 # Parquet


Test patch:
Repository: modin-project/modin

Issue:
Improve Dask performance by tuning `Client` parameters

### Describe the problem
<!-- Describe the problem clearly here. -->
Dask generally performs well, but it could be improved. I played around with changing `n_workers` as a parameter to `Client` and I found that the default does not take full advantage of the hardware in many cases.

### Source code / logs

To update here: 
https://github.com/modin-project/modin/blob/898889c8ac8804b7eed4cb3d733ee0d1e29686a8/modin/pandas/__init__.py#L227
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -223,9 +223,10 @@ def move_stdlib_ahead_of_site_packages(*args):
         client = _get_global_client()
         if client is None:
             from distributed import Client
+            import multiprocessing
 
-            client = Client()
-        num_cpus = sum(client.ncores().values())
+            num_cpus = multiprocessing.cpu_count()
+            client = Client(n_workers=num_cpus)
 elif execution_engine != "Python":
     raise ImportError("Unrecognized execution engine: {}.".format(execution_engine))
 


Test patch:
Repository: modin-project/modin

Issue:
Make __repr__ fit the shell width like pandas

### Describe the problem
<!-- Describe the problem clearly here. -->
Sometimes, the Modin DataFrame `__repr__` does not match the width of the window, but pandas does. This is a simple visualization issue and does not affect functionality.


Discussion:

Resolving patch:
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -88,9 +88,28 @@ def __init__(
             self._query_compiler = query_compiler
 
     def __repr__(self):
-        num_rows = pandas.get_option("max_rows") or 10
-        num_cols = pandas.get_option("max_columns") or 20
+        from pandas.io.formats import console
 
+        num_rows = pandas.get_option("display.max_rows") or 10
+        num_cols = pandas.get_option("display.max_columns") or 20
+        if pandas.get_option("display.max_columns") is None and pandas.get_option(
+            "display.expand_frame_repr"
+        ):
+            width, _ = console.get_console_size()
+            col_counter = 0
+            i = 0
+            while col_counter < width:
+                col_counter += len(str(self.columns[i])) + 1
+                i += 1
+
+            num_cols = i
+            i = len(self.columns) - 1
+            col_counter = 0
+            while col_counter < width:
+                col_counter += len(str(self.columns[i])) + 1
+                i -= 1
+
+            num_cols += len(self.columns) - i
         result = repr(self._build_repr_df(num_rows, num_cols))
         if len(self.index) > num_rows or len(self.columns) > num_cols:
             # The split here is so that we don't repr pandas row lengths.


Test patch:
Repository: modin-project/modin

Issue:
New API: `at_time` for Series and DataFrame

### Describe the problem
<!-- Describe the problem clearly here. -->
It should be relatively easy to add implementation for `at_time` for both `Series` and `DataFrame`. It is a filter on the index, so it should work well with existing functionality.


Discussion:

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -634,7 +634,10 @@ def at(self, axis=None):
         return _LocIndexer(self)
 
     def at_time(self, time, asof=False, axis=None):
-        return self._default_to_pandas("at_time", time, asof=asof, axis=axis)
+        axis = self._get_axis_number(axis) if axis is not None else 0
+        if axis == 0:
+            return self.iloc[self.index.indexer_at_time(time, asof=asof)]
+        return self.iloc[:, self.columns.indexer_at_time(time, asof=asof)]
 
     def between_time(
         self, start_time, end_time, include_start=True, include_end=True, axis=None


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -2164,10 +2164,18 @@ def test_assign(self):
         df_equals(modin_result, pandas_result)
 
     def test_at_time(self):
-        i = pd.date_range("2018-04-09", periods=4, freq="12H")
-        ts = pd.DataFrame({"A": [1, 2, 3, 4]}, index=i)
-        with pytest.warns(UserWarning):
-            ts.at_time("12:00")
+        i = pd.date_range("2008-01-01", periods=1000, freq="12H")
+        modin_df = pd.DataFrame(
+            {"A": list(range(1000)), "B": list(range(1000))}, index=i
+        )
+        pandas_df = pandas.DataFrame(
+            {"A": list(range(1000)), "B": list(range(1000))}, index=i
+        )
+        df_equals(modin_df.at_time("12:00"), pandas_df.at_time("12:00"))
+        df_equals(modin_df.at_time("3:00"), pandas_df.at_time("3:00"))
+        df_equals(
+            modin_df.T.at_time("12:00", axis=1), pandas_df.T.at_time("12:00", axis=1)
+        )
 
     def test_between_time(self):
         i = pd.date_range("2018-04-09", periods=4, freq="12H")
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -821,10 +821,11 @@ def test_at(data):
 
 
 def test_at_time():
-    i = pd.date_range("2018-04-09", periods=4, freq="12H")
-    ts = pd.Series([1, 2, 3, 4], index=i)
-    with pytest.warns(UserWarning):
-        ts.at_time("12:00")
+    i = pd.date_range("2008-01-01", periods=1000, freq="12H")
+    modin_series = pd.Series(list(range(1000)), index=i)
+    pandas_series = pandas.Series(list(range(1000)), index=i)
+    df_equals(modin_series.at_time("12:00"), pandas_series.at_time("12:00"))
+    df_equals(modin_series.at_time("3:00"), pandas_series.at_time("3:00"))
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)