internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: modin-project/modin

Issue:
Number of threads per Dask worker
We should add a configuration value for setting a number of threads per Dask worker.



Discussion:

Resolving patch:
diff --git a/modin/config/__init__.py b/modin/config/__init__.py
--- a/modin/config/__init__.py
+++ b/modin/config/__init__.py
@@ -21,6 +21,7 @@
     CIAWSAccessKeyID,
     CIAWSSecretAccessKey,
     CpuCount,
+    DaskThreadsPerWorker,
     DoUseCalcite,
     Engine,
     EnvironmentVariable,
@@ -73,6 +74,8 @@
     "RayRedisPassword",
     "TestRayClient",
     "LazyExecution",
+    # Dask specific
+    "DaskThreadsPerWorker",
     # Partitioning
     "NPartitions",
     "MinPartitionSize",
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -830,6 +830,13 @@ class LazyExecution(EnvironmentVariable, type=bool):
     default = False
 
 
+class DaskThreadsPerWorker(EnvironmentVariable, type=int):
+    """Number of threads per Dask worker."""
+
+    varname = "MODIN_DASK_THREADS_PER_WORKER"
+    default = None
+
+
 def _check_vars() -> None:
     """
     Check validity of environment variables.
diff --git a/modin/core/execution/dask/common/utils.py b/modin/core/execution/dask/common/utils.py
--- a/modin/core/execution/dask/common/utils.py
+++ b/modin/core/execution/dask/common/utils.py
@@ -19,6 +19,7 @@
     CIAWSAccessKeyID,
     CIAWSSecretAccessKey,
     CpuCount,
+    DaskThreadsPerWorker,
     GithubCI,
     Memory,
     NPartitions,
@@ -54,12 +55,17 @@ def _disable_warnings():
 """,
         )
         num_cpus = CpuCount.get()
+        threads_per_worker = DaskThreadsPerWorker.get()
         memory_limit = Memory.get()
         worker_memory_limit = memory_limit // num_cpus if memory_limit else "auto"
 
         # when the client is initialized, environment variables are inherited
         with set_env(PYTHONWARNINGS="ignore::FutureWarning"):
-            client = Client(n_workers=num_cpus, memory_limit=worker_memory_limit)
+            client = Client(
+                n_workers=num_cpus,
+                threads_per_worker=threads_per_worker,
+                memory_limit=worker_memory_limit,
+            )
 
         if GithubCI.get():
             # set these keys to run tests that write to the mock s3 service. this seems


Test patch:
Repository: modin-project/modin

Issue:
Add Support for `pydantic` 2
**Is your feature request related to a problem? Please describe.**
An explicit [restriction](https://github.com/modin-project/modin/issues/6336) was added to the requirements. However, `ray` seems to [support](https://github.com/ray-project/ray/blob/master/python/requirements.txt#L60) 2.5+ version of `pydantic`. Are there any more reasons to restrict it to 1.x?


Check out [this](https://github.com/ray-project/ray/commit/31c6631b8c5e098927c980aef195039c7f64fba0) commit which adds that functionality. It'll be really helpful if we could use `modin` with latest versions of `pydantic`.


Discussion:
Hi @achieveordie! I will try to unpin it.

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -7,8 +7,7 @@
 
 dask_deps = ["dask>=2.22.0", "distributed>=2.22.0"]
 # ray==2.5.0 broken: https://github.com/conda-forge/ray-packages-feedstock/issues/100
-# pydantic<2: https://github.com/modin-project/modin/issues/6336
-ray_deps = ["ray[default]>=1.13.0,!=2.5.0", "pyarrow>=7.0.0", "pydantic<2"]
+ray_deps = ["ray[default]>=1.13.0,!=2.5.0", "pyarrow>=7.0.0"]
 mpi_deps = ["unidist[mpi]>=0.2.1"]
 spreadsheet_deps = ["modin-spreadsheet>=0.1.0"]
 # Currently, Modin does not include `mpi` option in `all`.


Test patch:
Repository: modin-project/modin

Issue:
Remove testing for Ray client
[Ray Client](https://docs.ray.io/en/latest/cluster/running-applications/job-submission/ray-client.html#when-to-use-ray-client) has architectural limitations and may not work as expected in some scenarios. There are also long standing issues related to it in Ray repo, which are not fixed yet. We don't want to test the unstable package.



Discussion:

Resolving patch:
diff --git a/modin/config/__init__.py b/modin/config/__init__.py
--- a/modin/config/__init__.py
+++ b/modin/config/__init__.py
@@ -49,7 +49,6 @@
     ReadSqlEngine,
     StorageFormat,
     TestDatasetSize,
-    TestRayClient,
     TestReadFromPostgres,
     TestReadFromSqlServer,
     TrackFileLeaks,
@@ -71,7 +70,6 @@
     "IsRayCluster",
     "RayRedisAddress",
     "RayRedisPassword",
-    "TestRayClient",
     "LazyExecution",
     # Partitioning
     "NPartitions",
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -390,13 +390,6 @@ class TestDatasetSize(EnvironmentVariable, type=str):
     choices = ("Small", "Normal", "Big")
 
 
-class TestRayClient(EnvironmentVariable, type=bool):
-    """Set to true to start and connect Ray client before a testing session starts."""
-
-    varname = "MODIN_TEST_RAY_CLIENT"
-    default = False
-
-
 class TrackFileLeaks(EnvironmentVariable, type=bool):
     """Whether to track for open file handles leakage during testing."""
 
diff --git a/modin/conftest.py b/modin/conftest.py
--- a/modin/conftest.py
+++ b/modin/conftest.py
@@ -57,13 +57,10 @@ def _saving_make_api_url(token, _make_api_url=modin.utils._make_api_url):
 from modin.config import (  # noqa: E402
     AsyncReadMode,
     BenchmarkMode,
-    CIAWSAccessKeyID,
-    CIAWSSecretAccessKey,
     GithubCI,
     IsExperimental,
     MinPartitionSize,
     NPartitions,
-    TestRayClient,
 )
 from modin.core.execution.dispatching.factories import factories  # noqa: E402
 from modin.core.execution.python.implementations.pandas_on_python.io import (  # noqa: E402
@@ -496,31 +493,6 @@ def set_min_partition_size(request):
 ray_client_server = None
 
 
-def pytest_sessionstart(session):
-    if TestRayClient.get():
-        import ray
-        import ray.util.client.server.server as ray_server
-
-        addr = "localhost:50051"
-        global ray_client_server
-        ray_client_server = ray_server.serve(addr)
-        env_vars = {
-            "AWS_ACCESS_KEY_ID": CIAWSAccessKeyID.get(),
-            "AWS_SECRET_ACCESS_KEY": CIAWSSecretAccessKey.get(),
-        }
-        extra_init_kw = {"runtime_env": {"env_vars": env_vars}}
-        ray.util.connect(addr, ray_init_kwargs=extra_init_kw)
-
-
-def pytest_sessionfinish(session, exitstatus):
-    if TestRayClient.get():
-        import ray
-
-        ray.util.disconnect()
-        if ray_client_server:
-            ray_client_server.stop(0)
-
-
 @pytest.fixture
 def s3_storage_options(worker_id):
     # # copied from pandas conftest.py:


Test patch:
Repository: modin-project/modin

Issue:
Use pure multiprocessing for Dask
On a customer benchmark I am measuring `read_csv` is significantly slower with the default client rather than with the configured one.

```bash
read_csv: 13.72040319442749  # Client() # worker threads and processes take place
read_csv: 3.934723377227783  # Client(n_workers=16, threads_per_worker=1) # pure multiprocessing
```

We can set `DaskThreadsPerWorker` to 1 to get pure multiprocessing in Dask.


Discussion:

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -827,7 +827,7 @@ class DaskThreadsPerWorker(EnvironmentVariable, type=int):
     """Number of threads per Dask worker."""
 
     varname = "MODIN_DASK_THREADS_PER_WORKER"
-    default = None
+    default = 1
 
 
 def _check_vars() -> None:


Test patch:
Repository: modin-project/modin

Issue:
ASV: AttributeError: module 'pandas._testing' has no attribute 'makeStringIndex'



Discussion:

Resolving patch:
diff --git a/asv_bench/benchmarks/benchmarks.py b/asv_bench/benchmarks/benchmarks.py
--- a/asv_bench/benchmarks/benchmarks.py
+++ b/asv_bench/benchmarks/benchmarks.py
@@ -22,7 +22,6 @@
 import math
 
 import numpy as np
-import pandas._testing as tm
 
 from .utils import (
     GROUPBY_NGROUPS,
@@ -139,8 +138,10 @@ class TimeJoinStringIndex:
 
     def setup(self, shapes, sort):
         assert shapes[0] % 100 == 0, "implementation restriction"
-        level1 = tm.makeStringIndex(10).values
-        level2 = tm.makeStringIndex(shapes[0] // 100).values
+        level1 = IMPL.Index([f"i-{i}" for i in range(10)], dtype=object).values
+        level2 = IMPL.Index(
+            [f"i-{i}" for i in range(shapes[0] // 100)], dtype=object
+        ).values
         codes1 = np.arange(10).repeat(shapes[0] // 100)
         codes2 = np.tile(np.arange(shapes[0] // 100), 10)
         index2 = IMPL.MultiIndex(levels=[level1, level2], codes=[codes1, codes2])
@@ -897,8 +898,12 @@ def setup(self, shape):
         self.df2 = IMPL.DataFrame(
             index=range(rows), data=np.random.rand(rows, cols), columns=range(cols)
         )
-        level1 = tm.makeStringIndex(rows // 10).values.repeat(10)
-        level2 = np.tile(tm.makeStringIndex(10).values, rows // 10)
+        level1 = IMPL.Index(
+            [f"i-{i}" for i in range(rows // 10)], dtype=object
+        ).values.repeat(10)
+        level2 = np.tile(
+            IMPL.Index([f"i-{i}" for i in range(10)], dtype=object).values, rows // 10
+        )
         index = IMPL.MultiIndex.from_arrays([level1, level2])
         self.s = IMPL.Series(np.random.randn(rows), index=index)
         self.s_subset = self.s[::2]
@@ -1033,7 +1038,9 @@ def setup(self, shape):
         temp_df = DataFrame()
         # dataframe would  have cols-1 keys(strings) and one value(int) column
         for col in range(cols - 1):
-            temp_df["key" + str(col + 1)] = tm.makeStringIndex(N).values.repeat(K)
+            temp_df["key" + str(col + 1)] = IMPL.Index(
+                [f"i-{i}" for i in range(N)], dtype=object
+            ).values.repeat(K)
         self.df = IMPL.DataFrame(temp_df)
         self.df["value"] = np.random.randn(N * K)
         execute(self.df)
@@ -1052,7 +1059,12 @@ class TimeDropDuplicatesSeries:
 
     def setup(self, shape):
         rows = shape[0]
-        self.series = IMPL.Series(np.tile(tm.makeStringIndex(rows // 10).values, 10))
+        self.series = IMPL.Series(
+            np.tile(
+                IMPL.Index([f"i-{i}" for i in range(rows // 10)], dtype=object).values,
+                10,
+            )
+        )
         execute(self.series)
 
     def time_drop_dups(self, shape):


Test patch:
Repository: modin-project/modin

Issue:
isort check fails on master
https://github.com/modin-project/modin/actions/runs/7961357605/job/21732392100


Discussion:

Resolving patch:
diff --git a/examples/tutorial/jupyter/execution/pandas_on_ray/cluster/exercise_5.py b/examples/tutorial/jupyter/execution/pandas_on_ray/cluster/exercise_5.py
--- a/examples/tutorial/jupyter/execution/pandas_on_ray/cluster/exercise_5.py
+++ b/examples/tutorial/jupyter/execution/pandas_on_ray/cluster/exercise_5.py
@@ -1,4 +1,5 @@
 import time
+
 import ray
 
 import modin.pandas as pd


Test patch:
Repository: modin-project/modin

Issue:
Remove `DataFrame.to_pickle_distributed` in favour of `DataFrame.modin.to_pickle_distributed`



Discussion:

Resolving patch:
diff --git a/modin/experimental/pandas/__init__.py b/modin/experimental/pandas/__init__.py
--- a/modin/experimental/pandas/__init__.py
+++ b/modin/experimental/pandas/__init__.py
@@ -32,9 +32,6 @@
 >>> df = pd.read_csv_glob("data*.csv")
 """
 
-import functools
-import warnings
-
 from modin.pandas import *  # noqa F401, F403
 
 from .io import (  # noqa F401
@@ -45,20 +42,4 @@
     read_pickle_distributed,
     read_sql,
     read_xml_glob,
-    to_pickle_distributed,
 )
-
-old_to_pickle_distributed = to_pickle_distributed
-
-
-@functools.wraps(to_pickle_distributed)
-def to_pickle_distributed(*args, **kwargs):
-    warnings.warn(
-        "`DataFrame.to_pickle_distributed` is deprecated and will be removed in a future version. "
-        + "Please use `DataFrame.modin.to_pickle_distributed` instead.",
-        category=FutureWarning,
-    )
-    return old_to_pickle_distributed(*args, **kwargs)
-
-
-setattr(DataFrame, "to_pickle_distributed", to_pickle_distributed)  # noqa: F405


Test patch:
Repository: modin-project/modin

Issue:
Inaccessible https://discuss.modin.org
We stopped support for https://discuss.modin.org so we should remove links to it.


Discussion:

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -185,11 +185,6 @@ def noop_decorator(*args, **kwargs):
             "url": "https://modin.org/slack.html",
             "icon": "fab fa-slack",
         },
-        {
-            "name": "Discourse",
-            "url": "https://discuss.modin.org/",
-            "icon": "fab fa-discourse",
-        },
         {
             "name": "Mailing List",
             "url": "https://groups.google.com/forum/#!forum/modin-dev",


Test patch:
Repository: modin-project/modin

Issue:
Unnecessary `row_lengths` computation on `._propagate_index_objs(axis=1)`
<b>quote from [this comment](https://github.com/modin-project/modin/issues/6948#issuecomment-1969082652):</b>

The columns' propagation itself can be performed lazily, however the method explicitly calls empty partition filtering at the beginning that forcibly triggers computation for `row_lengths`:

https://github.com/modin-project/modin/blob/271d98b260f03b345ae2384001f2df83b98322bf/modin/core/dataframe/pandas/dataframe/dataframe.py#L883

This `._filter_empties()` call was introduced ~5 years ago by this PR (https://github.com/modin-project/modin/pull/721) and isn't really required for the indices' propagation to work. I suggest we replace this call with `._filter_empties(compute_metadata=False)` to not trigger computations here.


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -880,7 +880,7 @@ def _propagate_index_objs(self, axis=None):
         axis : int, default: None
             The axis to apply to. If it's None applies to both axes.
         """
-        self._filter_empties()
+        self._filter_empties(compute_metadata=False)
         if axis is None or axis == 0:
             cum_row_lengths = np.cumsum([0] + self.row_lengths)
         if axis is None or axis == 1:
@@ -931,7 +931,11 @@ def apply_idx_objs(df, idx):
                                 slice(cum_row_lengths[i], cum_row_lengths[i + 1])
                             ],
                             length=self.row_lengths[i],
-                            width=self.column_widths[j],
+                            width=(
+                                self.column_widths[j]
+                                if self._column_widths_cache is not None
+                                else None
+                            ),
                         )
                         for j in range(len(self._partitions[i]))
                     ]
@@ -952,7 +956,11 @@ def apply_idx_objs(df, cols):
                             cols=self.columns[
                                 slice(cum_col_widths[j], cum_col_widths[j + 1])
                             ],
-                            length=self.row_lengths[i],
+                            length=(
+                                self.row_lengths[i]
+                                if self._row_lengths_cache is not None
+                                else None
+                            ),
                             width=self.column_widths[j],
                         )
                         for j in range(len(self._partitions[i]))


Test patch:
Repository: modin-project/modin

Issue:
Update Exception message for `astype` function in the case of duplicated values



Discussion:

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -1005,11 +1005,7 @@ def astype(self, dtype, copy=None, errors="raise"):  # noqa: PR01, RT01, D200
         # convert it to a dict before passing it to the query compiler.
         if isinstance(dtype, (pd.Series, pandas.Series)):
             if not dtype.index.is_unique:
-                raise ValueError(
-                    "The new Series of types must have a unique index, i.e. "
-                    + "it must be one-to-one mapping from column names to "
-                    + " their new dtypes."
-                )
+                raise ValueError("cannot reindex on an axis with duplicate labels")
             dtype = {column: dtype for column, dtype in dtype.items()}
         # If we got a series or dict originally, dtype is a dict now. Its keys
         # must be column names.


Test patch:
diff --git a/modin/pandas/test/dataframe/test_map_metadata.py b/modin/pandas/test/dataframe/test_map_metadata.py
--- a/modin/pandas/test/dataframe/test_map_metadata.py
+++ b/modin/pandas/test/dataframe/test_map_metadata.py
@@ -439,6 +439,9 @@ def test_astype():
             if isinstance(df, pd.DataFrame)
             else pandas.Series([str, str], index=["col1", "col1"])
         ),
+        raising_exceptions=ValueError(
+            "cannot reindex on an axis with duplicate labels"
+        ),
     )
 
 
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -900,10 +900,18 @@ def execute_callable(fn, inplace=False, md_kwargs={}, pd_kwargs={}):
                 ), "Got Modin Exception type {}, but pandas Exception type {} was expected".format(
                     type(md_e), type(pd_e)
                 )
-                if raising_exceptions:
+                if raising_exceptions and isinstance(raising_exceptions, (list, tuple)):
                     assert not isinstance(
                         md_e, tuple(raising_exceptions)
                     ), f"not acceptable exception type: {md_e}"
+                elif raising_exceptions and type(raising_exceptions) is type:
+                    assert (
+                        type(md_e) is type(raising_exceptions)
+                        and md_e.args == raising_exceptions.args
+                    ), f"not acceptable Modin's exception: [{repr(md_e)}]"
+                    assert (
+                        pd_e.args == raising_exceptions.args
+                    ), f"not acceptable Pandas' exception: [{repr(pd_e)}]"
             else:
                 raise NoModinException(
                     f"Modin doesn't throw an exception, while pandas does: [{repr(pd_e)}]"
Repository: modin-project/modin

Issue:
Update Exception message for `shift` function in the case of axis = 1



Discussion:

Resolving patch:
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -1409,7 +1409,9 @@ def shift(
                 + f"{type(self).__name__}.shift"
             )
         if axis == 1:
-            raise ValueError(f"No axis named {axis} for object type {type(self)}")
+            raise ValueError(
+                f"No axis named {axis} for object type {type(self).__name__}"
+            )
         return super(type(self), self).shift(
             periods=periods, freq=freq, axis=axis, fill_value=fill_value
         )


Test patch:
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -3262,11 +3262,11 @@ def test_sum(data, skipna, numeric_only, min_count):
 
 @pytest.mark.parametrize("operation", ["sum", "shift"])
 def test_sum_axis_1_except(operation):
-    # ValueError('No axis named 1 for object type Series')
     eval_general(
         *create_test_series(test_data["int_data"]),
         lambda df, *args, **kwargs: getattr(df, operation)(*args, **kwargs),
         axis=1,
+        raising_exceptions=ValueError("No axis named 1 for object type Series"),
     )
 
 
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -904,7 +904,7 @@ def execute_callable(fn, inplace=False, md_kwargs={}, pd_kwargs={}):
                     assert not isinstance(
                         md_e, tuple(raising_exceptions)
                     ), f"not acceptable exception type: {md_e}"
-                elif raising_exceptions and type(raising_exceptions) is type:
+                elif raising_exceptions and type(raising_exceptions) is not type:
                     assert (
                         type(md_e) is type(raising_exceptions)
                         and md_e.args == raising_exceptions.args
Repository: modin-project/modin

Issue:
Update exception message for `iloc/loc` functions



Discussion:

Resolving patch:
diff --git a/modin/pandas/indexing.py b/modin/pandas/indexing.py
--- a/modin/pandas/indexing.py
+++ b/modin/pandas/indexing.py
@@ -290,9 +290,7 @@ def _validate_key_length(self, key: tuple) -> tuple:  # noqa: GL08
                 if Ellipsis in key:
                     raise IndexingError(_one_ellipsis_message)
                 return self._validate_key_length(key)
-            raise IndexingError(
-                f"Too many indexers: you're trying to pass {len(key)} indexers to the {type(self.df)} having only {self.df.ndim} dimensions."
-            )
+            raise IndexingError("Too many indexers")
         return key
 
     def __getitem__(self, key):  # pragma: no cover


Test patch:
diff --git a/modin/pandas/test/dataframe/test_indexing.py b/modin/pandas/test/dataframe/test_indexing.py
--- a/modin/pandas/test/dataframe/test_indexing.py
+++ b/modin/pandas/test/dataframe/test_indexing.py
@@ -870,10 +870,20 @@ def test_iloc_empty():
     df_equals(pandas_df, modin_df)
 
 
-def test_iloc_loc_key_length():
+def test_iloc_loc_key_length_except():
     modin_ser, pandas_ser = pd.Series(0), pandas.Series(0)
-    eval_general(modin_ser, pandas_ser, lambda ser: ser.iloc[0, 0])
-    eval_general(modin_ser, pandas_ser, lambda ser: ser.loc[0, 0])
+    eval_general(
+        modin_ser,
+        pandas_ser,
+        lambda ser: ser.iloc[0, 0],
+        raising_exceptions=pandas.errors.IndexingError("Too many indexers"),
+    )
+    eval_general(
+        modin_ser,
+        pandas_ser,
+        lambda ser: ser.loc[0, 0],
+        raising_exceptions=pandas.errors.IndexingError("Too many indexers"),
+    )
 
 
 def test_loc_series():
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -904,7 +904,7 @@ def execute_callable(fn, inplace=False, md_kwargs={}, pd_kwargs={}):
                     assert not isinstance(
                         md_e, tuple(raising_exceptions)
                     ), f"not acceptable exception type: {md_e}"
-                elif raising_exceptions and type(raising_exceptions) is type:
+                elif raising_exceptions and type(raising_exceptions) is not type:
                     assert (
                         type(md_e) is type(raising_exceptions)
                         and md_e.args == raising_exceptions.args