Repository: modin-project/modin

Issue:
Parallel agents for TeamCity
Right now, TeamCity is taking >5 hours to complete. We should create parallel agents for non-dependent builds to run concurrently. The breakdown should be as follows:

1.	`MODIN_ENGINE=ray`, `MODIN_ENGINE=dask`, `MODIN_ENGINE=python`
a.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameReduction_A`
b.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameReduction_B`
c.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameBinary`
d.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameMapMetadata`
e.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameUDF`
f.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameDefault`
g.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameWindow`
h.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameIndexing`
i.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameIter`
j.	`python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrameJoinSort`
k.	`python -m pytest modin/pandas/test/test_groupby.py`
l.	`python -m pytest modin/pandas/test/test_series.py modin/pandas/test/test_concat.py modin/pandas/test/test_reshape.py modin/pandas/test/test_general.py modin/pandas/test/test_io.py modin/pandas/test/test_io_exp.py`
2.	`MODIN_ENGINE=ray MODIN_EXPERIMENTAL=True MODIN_BACKEND=pyarrow python -m pytest modin/pandas/test/test_io.py::test_from_csv`

In total, 37 agents. This does not include agents we will need for Windows and MacOS.  



Discussion:

Resolving patch:
diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py
--- a/modin/pandas/__init__.py
+++ b/modin/pandas/__init__.py
@@ -75,6 +75,7 @@
 import os
 import types
 import sys
+import multiprocessing
 
 from .. import __version__
 from .concat import concat
@@ -134,12 +135,14 @@ def initialize_ray():
         import secrets
 
         plasma_directory = None
+        num_cpus = os.environ.get("MODIN_CPUS", None) or multiprocessing.cpu_count()
         cluster = os.environ.get("MODIN_RAY_CLUSTER", None)
         redis_address = os.environ.get("MODIN_REDIS_ADDRESS", None)
         redis_password = secrets.token_hex(16)
         if cluster == "True" and redis_address is not None:
             # We only start ray in a cluster setting for the head node.
             ray.init(
+                num_cpus=int(num_cpus),
                 include_webui=False,
                 ignore_reinit_error=True,
                 redis_address=redis_address,
@@ -171,6 +174,7 @@ def initialize_ray():
             else:
                 object_store_memory = int(object_store_memory)
             ray.init(
+                num_cpus=int(num_cpus),
                 include_webui=False,
                 ignore_reinit_error=True,
                 plasma_directory=plasma_directory,
@@ -229,10 +233,9 @@ def move_stdlib_ahead_of_site_packages(*args):
             client = get_client()
         except ValueError:
             from distributed import Client
-            import multiprocessing
 
-            num_cpus = multiprocessing.cpu_count()
-            client = Client(n_workers=num_cpus)
+            num_cpus = os.environ.get("MODIN_CPUS", None) or multiprocessing.cpu_count()
+            client = Client(n_workers=int(num_cpus))
 elif execution_engine != "Python":
     raise ImportError("Unrecognized execution engine: {}.".format(execution_engine))
 


Test patch:
diff --git a/df_test_requires.txt b/df_test_requires.txt
--- a/df_test_requires.txt
+++ b/df_test_requires.txt
@@ -4,6 +4,7 @@ psutil==5.4.8
 pytest
 pytest-testmon
 pytest-custom_exit_code
+pytest-xdist
 matplotlib
 xarray
 scipy
diff --git a/modin/pandas/test/test_api.py b/modin/pandas/test/test_api.py
--- a/modin/pandas/test/test_api.py
+++ b/modin/pandas/test/test_api.py
@@ -51,6 +51,10 @@ def test_top_level_api_equality():
         "num_cpus",
         "warnings",
         "os",
+        "multiprocessing",
+        "Client",
+        "client",
+        "get_client",
     ]
 
     assert not len(
diff --git a/windows_test_requires.txt b/windows_test_requires.txt
--- a/windows_test_requires.txt
+++ b/windows_test_requires.txt
@@ -15,6 +15,7 @@ coverage<5.0
 pytest-cov
 pytest-testmon
 pytest-custom_exit_code
+pytest-xdist
 feather-format
 lxml
 openpyxl