Repository: modin-project/modin

Issue:
Wrong number of partitions of a Modin DataFrame on a cluster
### Describe the problem
<!-- Describe the problem clearly here. -->

Details are in https://discuss.modin.org/t/dividing-a-modin-dataframe-into-partitions-on-a-cluster/183



Discussion:
This issue has been mentioned on **Modin Discuss**. There might be relevant details there:

https://discuss.modin.org/t/dividing-a-modin-dataframe-into-partitions-on-a-cluster/183/5


Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -123,7 +123,7 @@ class RayRedisAddress(EnvironmentVariable, type=ExactStr):
 
 class CpuCount(EnvironmentVariable, type=int):
     """
-    How may CPU cores to utilize across the whole distribution
+    How many CPU cores to use when initialization of the Modin engine.
     """
 
     varname = "MODIN_CPUS"
@@ -145,13 +145,34 @@ class Memory(EnvironmentVariable, type=int):
 
 class NPartitions(EnvironmentVariable, type=int):
     """
-    How many partitions to use by default
+    How many partitions to use for a Modin DataFrame (along each axis)
     """
 
     varname = "MODIN_NPARTITIONS"
+    # This flag is used to detect whether NPartitions is default value or not
+    _is_default = False
+
+    @classmethod
+    def put(cls, value):
+        cls._is_default = False
+        return super().put(value)
+
+    @classmethod
+    def put_if_default(cls, value):
+        """
+        Put specific value if NPartitions wasn't set by a user yet
+
+        Notes
+        -----
+        This method is used to set NPartitions from cluster resources internally
+        and should not be called by a user.
+        """
+        if cls._is_default:
+            cls.put(value)
 
     @classmethod
     def _get_default(cls):
+        cls._is_default = True
         return CpuCount.get()
 
 
diff --git a/modin/engines/dask/utils.py b/modin/engines/dask/utils.py
--- a/modin/engines/dask/utils.py
+++ b/modin/engines/dask/utils.py
@@ -11,7 +11,7 @@
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
-from modin.config import CpuCount
+from modin.config import CpuCount, NPartitions
 from modin.error_message import ErrorMessage
 
 
@@ -19,7 +19,7 @@ def initialize_dask():
     from distributed.client import get_client
 
     try:
-        get_client()
+        client = get_client()
     except ValueError:
         from distributed import Client
 
@@ -32,4 +32,7 @@ def initialize_dask():
     client = Client()
 """,
         )
-        Client(n_workers=CpuCount.get())
+        client = Client(n_workers=CpuCount.get())
+
+    num_cpus = len(client.ncores())
+    NPartitions.put_if_default(num_cpus)
diff --git a/modin/engines/ray/utils.py b/modin/engines/ray/utils.py
--- a/modin/engines/ray/utils.py
+++ b/modin/engines/ray/utils.py
@@ -22,6 +22,7 @@
     Memory,
     RayPlasmaDir,
     IsOutOfCore,
+    NPartitions,
 )
 
 
@@ -164,3 +165,6 @@ def initialize_ray(
         )
 
         ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
+
+    num_cpus = int(ray.cluster_resources()["CPU"])
+    NPartitions.put_if_default(num_cpus)


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -353,7 +353,16 @@ def test_read_csv_parsing_1(
     @pytest.mark.parametrize("skiprows", [2, lambda x: x % 2])
     @pytest.mark.parametrize("skipfooter", [0, 10])
     @pytest.mark.parametrize("nrows", [35, None])
-    @pytest.mark.parametrize("names", [["c1", "c2", "c3", "c4"], None])
+    @pytest.mark.parametrize(
+        "names",
+        [
+            pytest.param(
+                ["c1", "c2", "c3", "c4"],
+                marks=pytest.mark.xfail(reason="Excluded because of the issue #2845"),
+            ),
+            None,
+        ],
+    )
     def test_read_csv_parsing_2(
         self,
         request,
@@ -1835,6 +1844,10 @@ def test_to_stata(self):
 
 
 class TestFeather:
+    @pytest.mark.xfail(
+        Engine.get() != "Python",
+        reason="Excluded because of the issue #2845",
+    )
     def test_read_feather(self):
         unique_filename = get_unique_filename(extension="feather")
         try: