Repository: mars-project/mars

Issue:
Optimize Ray CI execution time
Currently Ray CI took at least 40~200 minutes to finish, which is very unfriendly to quick development and iteration. We should optimize it to 20 minutes at least.

After a quick took as ray ci execution stats, seems ray deploy took at least  20 minutes, which is enough for a separate CI pipeline. And if there are some unstable features introduced into mars on ray. A more lightweight will be helpful for debugging to. And CI retry wil take less time.
```
,timestamp,duration,stage,test,testfile,testname
0,2022-05-28T06:17:08.4237360Z,65.95,call,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_out[0-ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_out[0-ray_large_cluster0]
1,2022-05-28T06:17:08.4237705Z,50.65,call,mars/deploy/oscar/tests/test_ray.py::test_ray_client,mars/deploy/oscar/tests/test_ray.py,test_ray_client
2,2022-05-28T06:17:08.4238058Z,48.99,call,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_in[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_in[ray_large_cluster0]
4,2022-05-28T06:17:08.4238881Z,37.91,setup,mars/deploy/oscar/tests/test_ray_scheduling.py::test_task_speculation_execution[ray_large_cluster0],mars/deploy/oscar/tests/test_ray_scheduling.py,test_task_speculation_execution[ray_large_cluster0]
5,2022-05-28T06:17:08.4239261Z,34.01,call,mars/deploy/oscar/tests/test_ray.py::test_new_ray_session,mars/deploy/oscar/tests/test_ray.py,test_new_ray_session
6,2022-05-28T06:17:08.4239621Z,32.44,call,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option3],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option3]
7,2022-05-28T06:17:08.4239993Z,23.38,call,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option2],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option2]
9,2022-05-28T06:17:08.4240713Z,22.98,call,mars/deploy/oscar/tests/test_ray.py::test_new_cluster_in_ray,mars/deploy/oscar/tests/test_ray.py,test_new_cluster_in_ray
8,2022-05-28T06:17:08.4240360Z,22.98,call,mars/deploy/oscar/tests/test_ray.py::test_request_worker[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_request_worker[ray_large_cluster0]
11,2022-05-28T06:17:08.4241507Z,18.77,call,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules_from_config,mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules_from_config
12,2022-05-28T06:17:08.4241880Z,18.39,call,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option1],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option1]
13,2022-05-28T06:17:08.4242416Z,18.3,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_describe[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_describe[fault_config0-fault_cluster0]
14,2022-05-28T06:17:08.4242831Z,17.91,call,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option0],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option0]
15,2022-05-28T06:17:08.4243356Z,17.89,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_retryable[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_retryable[fault_config0-fault_cluster0]
16,2022-05-28T06:17:08.4243801Z,17.87,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception3],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception3]
17,2022-05-28T06:17:08.4244264Z,17.82,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception2],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception2]
18,2022-05-28T06:17:08.4244808Z,17.79,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config1-fault_cluster0]
20,2022-05-28T06:17:08.4245915Z,17.77,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config0-fault_cluster0]
19,2022-05-28T06:17:08.4245367Z,17.77,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_fail[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_fail[fault_config1-fault_cluster0]
21,2022-05-28T06:17:08.4246384Z,17.68,setup,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules2[create_cluster0],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules2[create_cluster0]
22,2022-05-28T06:17:08.4246902Z,17.66,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_fail[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_fail[fault_config0-fault_cluster0]
25,2022-05-28T06:17:08.4248384Z,17.43,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config2-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config2-fault_cluster0]
26,2022-05-28T06:17:08.4248744Z,17.41,setup,mars/deploy/oscar/tests/test_ray.py::test_fetch_infos,mars/deploy/oscar/tests/test_ray.py,test_fetch_infos
27,2022-05-28T06:17:08.4249254Z,17.35,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_describe[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_describe[fault_config1-fault_cluster0]
28,2022-05-28T06:17:08.4249716Z,17.25,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception0]
29,2022-05-28T06:17:08.4250171Z,17.25,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception1],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception1]
30,2022-05-28T06:17:08.4250549Z,17.15,setup,mars/deploy/oscar/tests/test_ray.py::test_execute[config1],mars/deploy/oscar/tests/test_ray.py,test_execute[config1]
31,2022-05-28T06:17:08.4250895Z,17.08,setup,mars/deploy/oscar/tests/test_ray.py::test_web_session[config1],mars/deploy/oscar/tests/test_ray.py,test_web_session[config1]
32,2022-05-28T06:17:08.4251238Z,17.04,setup,mars/deploy/oscar/tests/test_ray.py::test_init_metrics_on_ray,mars/deploy/oscar/tests/test_ray.py,test_init_metrics_on_ray
33,2022-05-28T06:17:08.4251563Z,17.01,setup,mars/deploy/oscar/tests/test_ray.py::test_execute_describe,mars/deploy/oscar/tests/test_ray.py,test_execute_describe
34,2022-05-28T06:17:08.4252057Z,17.0,setup,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_retryable[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_retryable[fault_config1-fault_cluster0]
35,2022-05-28T06:17:08.4252440Z,16.96,setup,mars/deploy/oscar/tests/test_ray.py::test_web_session[config0],mars/deploy/oscar/tests/test_ray.py,test_web_session[config0]
36,2022-05-28T06:17:08.4252781Z,16.89,setup,mars/deploy/oscar/tests/test_ray.py::test_web_session[config2],mars/deploy/oscar/tests/test_ray.py,test_web_session[config2]
37,2022-05-28T06:17:08.4253106Z,16.72,setup,mars/deploy/oscar/tests/test_ray.py::test_sync_execute,mars/deploy/oscar/tests/test_ray.py,test_sync_execute
38,2022-05-28T06:17:08.4253426Z,16.6,setup,mars/deploy/oscar/tests/test_ray.py::test_execute[config2],mars/deploy/oscar/tests/test_ray.py,test_execute[config2]
39,2022-05-28T06:17:08.4253757Z,16.48,setup,mars/deploy/oscar/tests/test_ray.py::test_iterative_tiling,mars/deploy/oscar/tests/test_ray.py,test_iterative_tiling
41,2022-05-28T06:17:08.4254482Z,16.44,setup,mars/deploy/oscar/tests/test_ray.py::test_execute[config0],mars/deploy/oscar/tests/test_ray.py,test_execute[config0]
42,2022-05-28T06:17:08.4255057Z,16.17,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_describe[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_describe[fault_config1-fault_cluster0]
44,2022-05-28T06:17:08.4255927Z,15.31,teardown,mars/deploy/oscar/tests/test_ray_scheduling.py::test_task_speculation_execution[ray_large_cluster0],mars/deploy/oscar/tests/test_ray_scheduling.py,test_task_speculation_execution[ray_large_cluster0]
45,2022-05-28T06:17:08.4256499Z,15.21,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_describe[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_describe[fault_config0-fault_cluster0]
46,2022-05-28T06:17:08.4257069Z,14.73,call,mars/deploy/oscar/tests/test_ray_dag_failover.py::test_basic_object_reconstruction[True-ray_large_cluster0],mars/deploy/oscar/tests/test_ray_dag_failover.py,test_basic_object_reconstruction[True-ray_large_cluster0]
47,2022-05-28T06:17:08.4257486Z,14.0,call,mars/deploy/oscar/tests/test_ray.py::test_reconstruct_worker[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_reconstruct_worker[ray_large_cluster0]
50,2022-05-28T06:17:08.4258891Z,11.58,call,mars/deploy/oscar/tests/test_ray_dag_failover.py::test_basic_object_reconstruction[False-ray_large_cluster0],mars/deploy/oscar/tests/test_ray_dag_failover.py,test_basic_object_reconstruction[False-ray_large_cluster0]
54,2022-05-28T06:17:08.4260579Z,9.6,call,mars/deploy/oscar/tests/test_ray.py::test_execute_describe,mars/deploy/oscar/tests/test_ray.py,test_execute_describe
58,2022-05-28T06:17:08.4262196Z,9.28,call,mars/deploy/oscar/tests/test_ray_dag.py::test_cancel[_cancel_when_execute],mars/deploy/oscar/tests/test_ray_dag.py,test_cancel[_cancel_when_execute]
59,2022-05-28T06:17:08.4262572Z,9.24,teardown,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules2[create_cluster0],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules2[create_cluster0]
60,2022-05-28T06:17:08.4263108Z,9.18,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config1-fault_cluster0]
66,2022-05-28T06:17:08.4265477Z,9.0,teardown,mars/deploy/oscar/tests/test_ray.py::test_execute_describe,mars/deploy/oscar/tests/test_ray.py,test_execute_describe
67,2022-05-28T06:17:08.4265886Z,8.98,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception2],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception2]
68,2022-05-28T06:17:08.4267107Z,8.89,teardown,mars/deploy/oscar/tests/test_ray.py::test_execute[config0],mars/deploy/oscar/tests/test_ray.py,test_execute[config0]
70,2022-05-28T06:17:08.4268066Z,8.84,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config0-fault_cluster0]
69,2022-05-28T06:17:08.4267510Z,8.84,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception1],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception1]
72,2022-05-28T06:17:08.4268870Z,8.8,teardown,mars/deploy/oscar/tests/test_ray.py::test_fetch_infos,mars/deploy/oscar/tests/test_ray.py,test_fetch_infos
73,2022-05-28T06:17:08.4269391Z,8.76,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_describe[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_describe[fault_config0-fault_cluster0]
74,2022-05-28T06:17:08.4269771Z,8.76,teardown,mars/deploy/oscar/tests/test_ray.py::test_web_session[config0],mars/deploy/oscar/tests/test_ray.py,test_web_session[config0]
75,2022-05-28T06:17:08.4270108Z,8.75,teardown,mars/deploy/oscar/tests/test_ray.py::test_web_session[config1],mars/deploy/oscar/tests/test_ray.py,test_web_session[config1]
76,2022-05-28T06:17:08.4270447Z,8.66,teardown,mars/deploy/oscar/tests/test_ray.py::test_iterative_tiling,mars/deploy/oscar/tests/test_ray.py,test_iterative_tiling
77,2022-05-28T06:17:08.4270777Z,8.61,teardown,mars/deploy/oscar/tests/test_ray.py::test_web_session[config2],mars/deploy/oscar/tests/test_ray.py,test_web_session[config2]
80,2022-05-28T06:17:08.4272077Z,8.49,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config2-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config2-fault_cluster0]
81,2022-05-28T06:17:08.4272455Z,8.48,teardown,mars/deploy/oscar/tests/test_ray.py::test_sync_execute,mars/deploy/oscar/tests/test_ray.py,test_sync_execute
83,2022-05-28T06:17:08.4273198Z,8.44,teardown,mars/deploy/oscar/tests/test_ray.py::test_init_metrics_on_ray,mars/deploy/oscar/tests/test_ray.py,test_init_metrics_on_ray
84,2022-05-28T06:17:08.4273599Z,8.42,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception0]
85,2022-05-28T06:17:08.4274231Z,8.42,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_fail[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_fail[fault_config1-fault_cluster0]
86,2022-05-28T06:17:08.4274784Z,8.42,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_fail[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_fail[fault_config0-fault_cluster0]
87,2022-05-28T06:17:08.4275320Z,8.33,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_retryable[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_retryable[fault_config0-fault_cluster0]
88,2022-05-28T06:17:08.4275756Z,8.19,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception3],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception3]
89,2022-05-28T06:17:08.4276138Z,8.19,teardown,mars/deploy/oscar/tests/test_ray.py::test_execute[config2],mars/deploy/oscar/tests/test_ray.py,test_execute[config2]
91,2022-05-28T06:17:08.4276999Z,7.97,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_retryable[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_retryable[fault_config1-fault_cluster0]
93,2022-05-28T06:17:08.4277803Z,7.91,teardown,mars/deploy/oscar/tests/test_ray.py::test_execute[config1],mars/deploy/oscar/tests/test_ray.py,test_execute[config1]
94,2022-05-28T06:17:08.4278310Z,7.82,teardown,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_describe[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_describe[fault_config1-fault_cluster0]
96,2022-05-28T06:17:08.4279258Z,7.73,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config1-fault_cluster0]
99,2022-05-28T06:17:08.4280409Z,7.63,call,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception2],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception2]
101,2022-05-28T06:17:08.4281149Z,7.54,call,mars/deploy/oscar/tests/test_ray_dag.py::test_sync_execute[config0],mars/deploy/oscar/tests/test_ray_dag.py,test_sync_execute[config0]
102,2022-05-28T06:17:08.4281547Z,7.44,call,mars/deploy/oscar/tests/test_ray_scheduling.py::test_task_speculation_execution[ray_large_cluster0],mars/deploy/oscar/tests/test_ray_scheduling.py,test_task_speculation_execution[ray_large_cluster0]
106,2022-05-28T06:17:08.4283206Z,6.51,call,mars/deploy/oscar/tests/test_ray.py::test_web_session[config0],mars/deploy/oscar/tests/test_ray.py,test_web_session[config0]
107,2022-05-28T06:17:08.4283532Z,6.17,call,mars/deploy/oscar/tests/test_ray.py::test_web_session[config2],mars/deploy/oscar/tests/test_ray.py,test_web_session[config2]
109,2022-05-28T06:17:08.4284222Z,5.99,call,mars/deploy/oscar/tests/test_ray.py::test_web_session[config1],mars/deploy/oscar/tests/test_ray.py,test_web_session[config1]
110,2022-05-28T06:17:08.4293052Z,5.96,call,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception1],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception1]
123,2022-05-28T06:17:08.4298977Z,5.27,setup,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception0],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception0]
136,2022-05-28T06:17:08.4304329Z,5.21,setup,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option2],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option2]
137,2022-05-28T06:17:08.4304881Z,5.19,setup,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_out[1-ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_out[1-ray_large_cluster0]
142,2022-05-28T06:17:08.4306934Z,5.1,setup,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_out[0-ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_out[0-ray_large_cluster0]
161,2022-05-28T06:17:08.4315055Z,4.49,setup,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception2],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception2]
164,2022-05-28T06:17:08.4316332Z,4.48,setup,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception1],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception1]
167,2022-05-28T06:17:08.4317713Z,4.45,setup,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules_from_config,mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules_from_config
168,2022-05-28T06:17:08.4318258Z,4.45,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_new_session_backend[new_session-ray],mars/deploy/oscar/tests/test_ray_dag.py,test_new_session_backend[new_session-ray]
169,2022-05-28T06:17:08.4318634Z,4.43,setup,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option3],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option3]
171,2022-05-28T06:17:08.4319347Z,4.41,setup,mars/deploy/oscar/tests/test_ray.py::test_ray_client,mars/deploy/oscar/tests/test_ray.py,test_ray_client
170,2022-05-28T06:17:08.4319003Z,4.41,setup,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_in[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_in[ray_large_cluster0]
173,2022-05-28T06:17:08.4320237Z,4.38,setup,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option0],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option0]
175,2022-05-28T06:17:08.4321009Z,4.36,setup,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option1],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option1]
177,2022-05-28T06:17:08.4321877Z,4.03,teardown,mars/deploy/oscar/tests/test_ray_dag_failover.py::test_basic_object_reconstruction[True-ray_large_cluster0],mars/deploy/oscar/tests/test_ray_dag_failover.py,test_basic_object_reconstruction[True-ray_large_cluster0]
178,2022-05-28T06:17:08.4322288Z,4.01,setup,mars/deploy/oscar/tests/test_ray.py::test_request_worker[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_request_worker[ray_large_cluster0]
179,2022-05-28T06:17:08.4322705Z,3.95,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception2],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception2]
180,2022-05-28T06:17:08.4323121Z,3.91,setup,mars/deploy/oscar/tests/test_ray.py::test_reconstruct_worker[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_reconstruct_worker[ray_large_cluster0]
182,2022-05-28T06:17:08.4324258Z,3.8,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_retryable[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_retryable[fault_config1-fault_cluster0]
185,2022-05-28T06:17:08.4325620Z,3.71,teardown,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option3],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option3]
186,2022-05-28T06:17:08.4326097Z,3.69,teardown,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_out[0-ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_out[0-ray_large_cluster0]
187,2022-05-28T06:17:08.4326466Z,3.68,teardown,mars/deploy/oscar/tests/test_ray.py::test_request_worker[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_request_worker[ray_large_cluster0]
192,2022-05-28T06:17:08.4329146Z,3.48,teardown,mars/deploy/oscar/tests/test_ray_dag_failover.py::test_basic_object_reconstruction[False-ray_large_cluster0],mars/deploy/oscar/tests/test_ray_dag_failover.py,test_basic_object_reconstruction[False-ray_large_cluster0]
196,2022-05-28T06:17:08.4330822Z,3.21,teardown,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception0],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception0]
197,2022-05-28T06:17:08.4331225Z,3.2,teardown,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception1],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception1]
209,2022-05-28T06:17:08.4336855Z,3.08,teardown,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option0],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option0]
210,2022-05-28T06:17:08.4337248Z,3.07,teardown,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception2],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception2]
219,2022-05-28T06:17:08.4340995Z,2.98,teardown,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_in[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_in[ray_large_cluster0]
222,2022-05-28T06:17:08.4342303Z,2.94,teardown,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option1],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option1]
224,2022-05-28T06:17:08.4343021Z,2.94,teardown,mars/deploy/oscar/tests/test_ray.py::test_ray_client,mars/deploy/oscar/tests/test_ray.py,test_ray_client
225,2022-05-28T06:17:08.4343378Z,2.93,teardown,mars/deploy/oscar/tests/test_ray.py::test_optional_supervisor_node[test_option2],mars/deploy/oscar/tests/test_ray.py,test_optional_supervisor_node[test_option2]
227,2022-05-28T06:17:08.4344188Z,2.93,teardown,mars/deploy/oscar/tests/test_ray.py::test_auto_scale_out[1-ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_auto_scale_out[1-ray_large_cluster0]
230,2022-05-28T06:17:08.4345403Z,2.85,teardown,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules_from_config,mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules_from_config
231,2022-05-28T06:17:08.4345772Z,2.83,teardown,mars/deploy/oscar/tests/test_ray.py::test_reconstruct_worker[ray_large_cluster0],mars/deploy/oscar/tests/test_ray.py,test_reconstruct_worker[ray_large_cluster0]
233,2022-05-28T06:17:08.4346762Z,2.53,call,mars/deploy/oscar/tests/test_ray_dag.py::test_new_session_backend[new_session-ray],mars/deploy/oscar/tests/test_ray_dag.py,test_new_session_backend[new_session-ray]
236,2022-05-28T06:17:08.4347925Z,2.1,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_cancel[_cancel_when_tile],mars/deploy/oscar/tests/test_ray_dag.py,test_cancel[_cancel_when_tile]
239,2022-05-28T06:17:08.4349099Z,1.95,teardown,mars/deploy/oscar/tests/test_ray.py::test_new_ray_session,mars/deploy/oscar/tests/test_ray.py,test_new_ray_session
240,2022-05-28T06:17:08.4349434Z,1.76,teardown,mars/deploy/oscar/tests/test_ray.py::test_new_cluster_in_ray,mars/deploy/oscar/tests/test_ray.py,test_new_cluster_in_ray
243,2022-05-28T06:17:08.4350603Z,1.56,call,mars/deploy/oscar/tests/test_ray_dag.py::test_cancel[_cancel_when_tile],mars/deploy/oscar/tests/test_ray_dag.py,test_cancel[_cancel_when_tile]
244,2022-05-28T06:17:08.4351112Z,1.45,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_fail[fault_config1-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_fail[fault_config1-fault_cluster0]
249,2022-05-28T06:17:08.4353513Z,1.18,call,mars/deploy/oscar/tests/test_ray_dag.py::test_session_get_progress,mars/deploy/oscar/tests/test_ray_dag.py,test_session_get_progress
252,2022-05-28T06:17:08.4354845Z,0.98,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config2-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config2-fault_cluster0]
253,2022-05-28T06:17:08.4355371Z,0.98,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask[fault_config0-fault_cluster0]
254,2022-05-28T06:17:08.4355748Z,0.93,call,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config1],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config1]
257,2022-05-28T06:17:08.4356805Z,0.66,call,mars/deploy/oscar/tests/test_ray.py::test_fetch_infos,mars/deploy/oscar/tests/test_ray.py,test_fetch_infos
258,2022-05-28T06:17:08.4357135Z,0.65,call,mars/deploy/oscar/tests/test_ray.py::test_iterative_tiling,mars/deploy/oscar/tests/test_ray.py,test_iterative_tiling
259,2022-05-28T06:17:08.4357454Z,0.58,call,mars/deploy/oscar/tests/test_ray.py::test_sync_execute,mars/deploy/oscar/tests/test_ray.py,test_sync_execute
261,2022-05-28T06:17:08.4358295Z,0.53,call,mars/deploy/oscar/tests/test_ray_dag.py::test_new_session_backend[new_test_session-ray],mars/deploy/oscar/tests/test_ray_dag.py,test_new_session_backend[new_test_session-ray]
263,2022-05-28T06:17:08.4359120Z,0.49,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception0]
265,2022-05-28T06:17:08.4360138Z,0.47,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_retryable[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_retryable[fault_config0-fault_cluster0]
267,2022-05-28T06:17:08.4360961Z,0.38,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config2],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config2]
268,2022-05-28T06:17:08.4361332Z,0.36,call,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules2[create_cluster0],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules2[create_cluster0]
269,2022-05-28T06:17:08.4361696Z,0.33,call,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config0],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config0]
270,2022-05-28T06:17:08.4362034Z,0.31,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_iterative_tiling,mars/deploy/oscar/tests/test_ray_dag.py,test_iterative_tiling
273,2022-05-28T06:17:08.4363321Z,0.3,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception1],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception1]
275,2022-05-28T06:17:08.4364003Z,0.3,call,mars/deploy/oscar/tests/test_ray.py::test_execute[config1],mars/deploy/oscar/tests/test_ray.py,test_execute[config1]
274,2022-05-28T06:17:08.4363701Z,0.3,call,mars/deploy/oscar/tests/test_ray.py::test_execute[config2],mars/deploy/oscar/tests/test_ray.py,test_execute[config2]
272,2022-05-28T06:17:08.4362867Z,0.3,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_rerun_subtask_fail[fault_config0-fault_cluster0],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_rerun_subtask_fail[fault_config0-fault_cluster0]
271,2022-05-28T06:17:08.4362361Z,0.3,call,mars/deploy/oscar/tests/test_ray.py::test_execute[config0],mars/deploy/oscar/tests/test_ray.py,test_execute[config0]
276,2022-05-28T06:17:08.4364333Z,0.27,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_session_get_progress,mars/deploy/oscar/tests/test_ray_dag.py,test_session_get_progress
279,2022-05-28T06:17:08.4365485Z,0.18,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_cancel[_cancel_when_execute],mars/deploy/oscar/tests/test_ray_dag.py,test_cancel[_cancel_when_execute]
280,2022-05-28T06:17:08.4365839Z,0.13,call,mars/deploy/oscar/tests/test_ray_dag.py::test_iterative_tiling,mars/deploy/oscar/tests/test_ray_dag.py,test_iterative_tiling
281,2022-05-28T06:17:08.4366233Z,0.13,call,mars/deploy/oscar/tests/test_ray_fault_injection.py::test_fault_inject_subtask_processor[fault_and_exception3],mars/deploy/oscar/tests/test_ray_fault_injection.py,test_fault_inject_subtask_processor[fault_and_exception3]
284,2022-05-28T06:17:08.4367394Z,0.12,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config0],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config0]
289,2022-05-28T06:17:08.4369396Z,0.09,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config2],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config2]
290,2022-05-28T06:17:08.4369741Z,0.08,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_session_get_progress,mars/deploy/oscar/tests/test_ray_dag.py,test_session_get_progress
291,2022-05-28T06:17:08.4370083Z,0.08,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_iterative_tiling,mars/deploy/oscar/tests/test_ray_dag.py,test_iterative_tiling
294,2022-05-28T06:17:08.4371085Z,0.07,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config1],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config1]
296,2022-05-28T06:17:08.4371773Z,0.07,call,mars/deploy/oscar/tests/test_ray.py::test_load_config,mars/deploy/oscar/tests/test_ray.py,test_load_config
295,2022-05-28T06:17:08.4371436Z,0.07,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_cancel[_cancel_when_execute],mars/deploy/oscar/tests/test_ray_dag.py,test_cancel[_cancel_when_execute]
293,2022-05-28T06:17:08.4370767Z,0.07,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config0],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config0]
292,2022-05-28T06:17:08.4370431Z,0.07,setup,mars/deploy/oscar/tests/test_ray_dag.py::test_cancel[_cancel_when_tile],mars/deploy/oscar/tests/test_ray_dag.py,test_cancel[_cancel_when_tile]
297,2022-05-28T06:17:08.4372140Z,0.06,call,mars/deploy/oscar/tests/test_ray.py::test_load_third_party_modules[config_exception0],mars/deploy/oscar/tests/test_ray.py,test_load_third_party_modules[config_exception0]
298,2022-05-28T06:17:08.4372497Z,0.06,call,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config2],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config2]
307,2022-05-28T06:17:08.4376069Z,0.03,teardown,mars/deploy/oscar/tests/test_ray_dag.py::test_execute[config1],mars/deploy/oscar/tests/test_ray_dag.py,test_execute[config1]
322,2022-05-28T06:17:08.4382575Z,0.01,call,mars/deploy/oscar/tests/test_ray.py::test_reconstruct_worker_during_releasing_worker,mars/deploy/oscar/tests/test_ray.py,test_reconstruct_worker_during_releasing_worker

```


Discussion:

Resolving patch:
diff --git a/mars/conftest.py b/mars/conftest.py
--- a/mars/conftest.py
+++ b/mars/conftest.py
@@ -165,7 +165,6 @@ async def ray_create_mars_cluster(request, check_router_cleaned):
     worker_mem = param.get("worker_mem", 256 * 1024**2)
     ray_config.update(param.get("config", {}))
     client = await new_cluster(
-        "test_cluster",
         supervisor_mem=supervisor_mem,
         worker_num=worker_num,
         worker_cpu=worker_cpu,
diff --git a/mars/oscar/backends/ray/driver.py b/mars/oscar/backends/ray/driver.py
--- a/mars/oscar/backends/ray/driver.py
+++ b/mars/oscar/backends/ray/driver.py
@@ -64,7 +64,9 @@ def stop_cluster(cls):
         pg_name = cls._cluster_info["pg_name"]
         pg = cls._cluster_info["pg_group"]
         for index, bundle_spec in enumerate(pg.bundle_specs):
-            n_process = int(bundle_spec["CPU"]) + 1
+            # Main pool took a process.
+            # If supervisor is created in the same node with worker, it will take a process too.
+            n_process = int(bundle_spec["CPU"]) + 2
             for process_index in reversed(range(n_process)):
                 address = process_placement_to_address(
                     pg_name, index, process_index=process_index


Test patch:
diff --git a/mars/deploy/oscar/tests/test_ray.py b/mars/deploy/oscar/tests/test_ray.py
--- a/mars/deploy/oscar/tests/test_ray.py
+++ b/mars/deploy/oscar/tests/test_ray.py
@@ -16,30 +16,16 @@
 import copy
 import operator
 import os
-import subprocess
-import sys
-import tempfile
-import threading
-import time
 from functools import reduce
 
 import numpy as np
-import pandas as pd
 import pytest
 
 import mars
-from .... import oscar as mo
 from .... import tensor as mt
 from .... import dataframe as md
-from ....oscar.backends.ray.utils import (
-    process_address_to_placement,
-    process_placement_to_address,
-    kill_and_wait,
-)
 from ....oscar.errors import ReconstructWorkerError
 from ....serialization.ray import register_ray_serializers
-from ....services.cluster import ClusterAPI
-from ....services.scheduling.supervisor.autoscale import AutoscalerActor
 from ....tests.core import require_ray, mock, DICT_NOT_EMPTY
 from ....utils import lazy_import
 from ..ray import (
@@ -98,7 +84,6 @@ async def create_cluster(request):
     ray_config = _load_config(CONFIG_FILE)
     ray_config.update(param.get("config", {}))
     client = await new_cluster(
-        "test_cluster",
         supervisor_mem=1 * 1024**3,
         worker_num=2,
         worker_cpu=2,
@@ -135,25 +120,25 @@ async def create_cluster(request):
     ],
 )
 @pytest.mark.asyncio
-async def test_execute(ray_start_regular, create_cluster, config):
+async def test_execute(ray_start_regular_shared, create_cluster, config):
     await test_local.test_execute(create_cluster, config)
 
 
 @require_ray
 @pytest.mark.asyncio
-async def test_iterative_tiling(ray_start_regular, create_cluster):
+async def test_iterative_tiling(ray_start_regular_shared, create_cluster):
     await test_local.test_iterative_tiling(create_cluster)
 
 
 @require_ray
 @pytest.mark.asyncio
-async def test_execute_describe(ray_start_regular, create_cluster):
+async def test_execute_describe(ray_start_regular_shared, create_cluster):
     await test_local.test_execute_describe(create_cluster)
 
 
 @require_ray
 @pytest.mark.asyncio
-async def test_fetch_infos(ray_start_regular, create_cluster):
+async def test_fetch_infos(ray_start_regular_shared, create_cluster):
     await test_local.test_fetch_infos(create_cluster)
     df = md.DataFrame(mt.random.RandomState(0).rand(5000, 1, chunk_size=1000))
     df.execute()
@@ -165,7 +150,7 @@ async def test_fetch_infos(ray_start_regular, create_cluster):
 
 @require_ray
 @pytest.mark.asyncio
-def test_sync_execute(ray_start_regular, create_cluster):
+def test_sync_execute(ray_start_regular_shared, create_cluster):
     client = create_cluster[0]
     assert client.session
     session = new_session(address=client.address)
@@ -246,59 +231,6 @@ def new_ray_session_test():
     mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
 
 
-@require_ray
-def test_ray_client(ray_start_regular):
-    server_code = """import time
-import ray.util.client.server.server as ray_client_server
-
-server = ray_client_server.init_and_serve("{address}", num_cpus=20)
-print("OK", flush=True)
-while True:
-    time.sleep(1)
-"""
-
-    address = "127.0.0.1:50051"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as f:
-        f.write(server_code.format(address=address))
-        f.flush()
-
-        proc = subprocess.Popen([sys.executable, "-u", f.name], stdout=subprocess.PIPE)
-
-        try:
-
-            def _check_ready(expect_exit=False):
-                while True:
-                    line = proc.stdout.readline()
-                    if proc.returncode is not None:
-                        if expect_exit:
-                            break
-                        raise Exception(
-                            f"Failed to start ray server at {address}, "
-                            f"the return code is {proc.returncode}."
-                        )
-                    if b"OK" in line:
-                        break
-
-            # Avoid ray.init timeout.
-            _check_ready()
-
-            # Avoid blocking the subprocess when the stdout pipe is full.
-            t = threading.Thread(target=_check_ready, args=(True,))
-            t.start()
-
-            ray.init(f"ray://{address}")
-            ray._inside_client_test = True
-            try:
-                new_ray_session_test()
-            finally:
-                ray._inside_client_test = False
-                ray.shutdown()
-        finally:
-            proc.kill()
-            proc.wait()
-
-
 @require_ray
 @pytest.mark.parametrize(
     "test_option",
@@ -310,7 +242,7 @@ def _check_ready(expect_exit=False):
     ],
 )
 @pytest.mark.asyncio
-async def test_optional_supervisor_node(ray_start_regular, test_option):
+async def test_optional_supervisor_node(ray_start_regular_shared, test_option):
     import logging
 
     logging.basicConfig(level=logging.INFO)
@@ -357,7 +289,7 @@ async def test_optional_supervisor_node(ray_start_regular, test_option):
     ],
 )
 @pytest.mark.asyncio
-async def test_web_session(ray_start_regular, create_cluster, config):
+async def test_web_session(ray_start_regular_shared, create_cluster, config):
     client = create_cluster[0]
     await test_local.test_web_session(create_cluster, config)
     web_address = client.web_address
@@ -381,15 +313,15 @@ async def test_web_session(ray_start_regular, create_cluster, config):
     ],
 )
 @pytest.mark.asyncio
-async def test_load_third_party_modules(ray_start_regular, config_exception):
+async def test_load_third_party_modules(ray_start_regular_shared, config_exception):
     third_party_modules_config, expected_exception = config_exception
     config = _load_config()
 
     config["third_party_modules"] = third_party_modules_config
     with expected_exception:
         await new_cluster(
-            worker_num=2,
-            worker_cpu=2,
+            worker_num=1,
+            worker_cpu=1,
             worker_mem=1 * 1024**3,
             config=config,
         )
@@ -410,7 +342,7 @@ async def test_load_third_party_modules(ray_start_regular, config_exception):
     indirect=True,
 )
 @pytest.mark.asyncio
-def test_load_third_party_modules2(ray_start_regular, create_cluster):
+def test_load_third_party_modules2(ray_start_regular_shared, create_cluster):
     client = create_cluster[0]
     assert client.session
     session = new_session(address=client.address)
@@ -429,18 +361,18 @@ def test_load_third_party_modules2(ray_start_regular, create_cluster):
 @require_ray
 @pytest.mark.asyncio
 async def test_load_third_party_modules_from_config(
-    ray_start_regular, cleanup_third_party_modules_output  # noqa: F811
+    ray_start_regular_shared, cleanup_third_party_modules_output  # noqa: F811
 ):
     client = await new_cluster(
         supervisor_mem=1 * 1024**3,
-        worker_num=2,
-        worker_cpu=2,
+        worker_num=1,
+        worker_cpu=1,
         worker_mem=1 * 1024**3,
         config=CONFIG_THIRD_PARTY_MODULES_TEST_FILE,
     )
     async with client:
-        # 1 supervisor, 2 worker main pools, 4 worker sub pools.
-        assert len(get_output_filenames()) == 7
+        # 1 supervisor, 1 worker main pools, 1 worker sub pools.
+        assert len(get_output_filenames()) == 3
 
 
 @require_ray
@@ -462,84 +394,6 @@ def test_load_config():
     assert _load_config(CONFIG_FILE)["session"]["custom_log_dir"] == "auto"
 
 
-@pytest.mark.parametrize(
-    "ray_large_cluster", [{"num_nodes": 1, "num_cpus": 3}], indirect=True
-)
-@require_ray
-@pytest.mark.asyncio
-async def test_request_worker(ray_large_cluster):
-    worker_cpu, worker_mem = 1, 100 * 1024**2
-    client = await new_cluster(
-        worker_num=0, worker_cpu=worker_cpu, worker_mem=worker_mem
-    )
-    async with client:
-        cluster_state_ref = client._cluster._cluster_backend.get_cluster_state_ref()
-        # Note that supervisor took one node
-        workers = await asyncio.gather(
-            *[cluster_state_ref.request_worker(timeout=5) for _ in range(2)]
-        )
-        assert all(worker is not None for worker in workers)
-        assert not await cluster_state_ref.request_worker(timeout=5)
-        release_workers = [
-            cluster_state_ref.release_worker(worker) for worker in workers
-        ]
-        # Duplicate release workers requests should be handled.
-        release_workers.extend(
-            [cluster_state_ref.release_worker(worker) for worker in workers]
-        )
-        await asyncio.gather(*release_workers)
-        assert await cluster_state_ref.request_worker(timeout=5)
-        cluster_state_ref.reconstruct_worker()
-
-
-@pytest.mark.parametrize(
-    "ray_large_cluster", [{"num_nodes": 1, "num_cpus": 3}], indirect=True
-)
-@require_ray
-@pytest.mark.asyncio
-async def test_reconstruct_worker(ray_large_cluster):
-    worker_cpu, worker_mem = 1, 100 * 1024**2
-    client = await new_cluster(
-        worker_num=0, worker_cpu=worker_cpu, worker_mem=worker_mem
-    )
-    async with client:
-        cluster_api = await ClusterAPI.create(client._cluster.supervisor_address)
-        worker = await cluster_api.request_worker(timeout=5)
-        pg_name, bundle_index, process_index = process_address_to_placement(worker)
-        worker_sub_pool = process_placement_to_address(
-            pg_name, bundle_index, process_index + 1
-        )
-
-        worker_actor = ray.get_actor(worker)
-        worker_pid = await worker_actor.getpid.remote()
-        # the worker pool actor should be destroyed even we get actor.
-        worker_sub_pool_actor = ray.get_actor(worker_sub_pool)
-        worker_sub_pool_pid = await worker_sub_pool_actor.getpid.remote()
-
-        # kill worker main pool
-        await kill_and_wait(ray.get_actor(worker))
-
-        # duplicated reconstruct worker request can be handled.
-        await asyncio.gather(
-            cluster_api.reconstruct_worker(worker),
-            cluster_api.reconstruct_worker(worker),
-        )
-        worker_actor = ray.get_actor(worker)
-        new_worker_pid = await worker_actor.getpid.remote()
-        worker_sub_pool_actor = ray.get_actor(worker_sub_pool)
-        new_worker_sub_pool_pid = await worker_sub_pool_actor.getpid.remote()
-        assert new_worker_pid != worker_pid
-        assert new_worker_sub_pool_pid != worker_sub_pool_pid
-
-        # the compute should be ok after the worker is reconstructed.
-        raw = np.random.RandomState(0).rand(10, 5)
-        a = mt.tensor(raw, chunk_size=5).sum(axis=1)
-        b = a.execute(show_progress=False)
-        assert b is a
-        result = a.fetch()
-        np.testing.assert_array_equal(result, raw.sum(axis=1))
-
-
 @require_ray
 @pytest.mark.asyncio
 @mock.patch("mars.deploy.oscar.ray.stop_worker")
@@ -595,138 +449,9 @@ async def _stop_worker(*args):
     release_task.cancel()
 
 
-@pytest.mark.parametrize(
-    "ray_large_cluster", [{"num_nodes": 2, "num_cpus": 4}], indirect=True
-)
-@pytest.mark.parametrize("init_workers", [0, 1])
-@require_ray
-@pytest.mark.asyncio
-async def test_auto_scale_out(ray_large_cluster, init_workers: int):
-    client = await new_cluster(
-        worker_num=init_workers,
-        worker_cpu=2,
-        worker_mem=200 * 1024**2,
-        supervisor_mem=1 * 1024**3,
-        config={
-            "scheduling.autoscale.enabled": True,
-            "scheduling.autoscale.scheduler_backlog_timeout": 1,
-            "scheduling.autoscale.worker_idle_timeout": 10000000,
-            "scheduling.autoscale.max_workers": 10,
-        },
-    )
-    async with client:
-
-        def time_consuming(x):
-            time.sleep(1)
-            return x * x
-
-        series_size = 100
-        assert (
-            md.Series(list(range(series_size)), chunk_size=1)
-            .apply(time_consuming)
-            .sum()
-            .execute()
-            .fetch()
-            == pd.Series(list(range(series_size))).apply(lambda x: x * x).sum()
-        )
-        autoscaler_ref = mo.create_actor_ref(
-            uid=AutoscalerActor.default_uid(),
-            address=client._cluster.supervisor_address,
-        )
-        assert await autoscaler_ref.get_dynamic_worker_nums() > 0
-
-
-@pytest.mark.timeout(timeout=600)
-@pytest.mark.parametrize(
-    "ray_large_cluster", [{"num_nodes": 2, "num_cpus": 4}], indirect=True
-)
-@require_ray
-@pytest.mark.asyncio
-async def test_auto_scale_in(ray_large_cluster):
-    config = _load_config()
-    config["scheduling"]["autoscale"]["enabled"] = True
-    config["scheduling"]["autoscale"]["worker_idle_timeout"] = 1
-    config["scheduling"]["autoscale"]["max_workers"] = 4
-    config["scheduling"]["autoscale"]["min_workers"] = 2
-    client = await new_cluster(
-        worker_num=0,
-        worker_cpu=2,
-        worker_mem=200 * 102**2,
-        supervisor_mem=1 * 1024**3,
-        config=config,
-    )
-    async with client:
-        autoscaler_ref = mo.create_actor_ref(
-            uid=AutoscalerActor.default_uid(),
-            address=client._cluster.supervisor_address,
-        )
-        new_worker_nums = 3
-        await asyncio.gather(
-            *[autoscaler_ref.request_worker() for _ in range(new_worker_nums)]
-        )
-        series_size = 100
-        assert (
-            md.Series(list(range(series_size)), chunk_size=20).sum().execute().fetch()
-            == pd.Series(list(range(series_size))).sum()
-        )
-        while await autoscaler_ref.get_dynamic_worker_nums() > 2:
-            dynamic_workers = await autoscaler_ref.get_dynamic_workers()
-            print(f"Waiting workers {dynamic_workers} to be released.")
-            await asyncio.sleep(1)
-        await asyncio.sleep(1)
-        assert await autoscaler_ref.get_dynamic_worker_nums() == 2
-
-
-@pytest.mark.skip("Enable it when ray ownership bug is fixed")
-@pytest.mark.timeout(timeout=200)
-@pytest.mark.parametrize("ray_large_cluster", [{"num_nodes": 4}], indirect=True)
-@require_ray
-@pytest.mark.asyncio
-async def test_ownership_when_scale_in(ray_large_cluster):
-    client = await new_cluster(
-        worker_num=0,
-        worker_cpu=2,
-        worker_mem=200 * 1024**2,
-        supervisor_mem=200 * 1024**2,
-        config={
-            "scheduling.autoscale.enabled": True,
-            "scheduling.autoscale.scheduler_check_interval": 1,
-            "scheduling.autoscale.scheduler_backlog_timeout": 1,
-            "scheduling.autoscale.worker_idle_timeout": 10,
-            "scheduling.autoscale.min_workers": 1,
-            "scheduling.autoscale.max_workers": 4,
-        },
-    )
-    async with client:
-        autoscaler_ref = mo.create_actor_ref(
-            uid=AutoscalerActor.default_uid(),
-            address=client._cluster.supervisor_address,
-        )
-        await asyncio.gather(*[autoscaler_ref.request_worker() for _ in range(2)])
-        df = md.DataFrame(mt.random.rand(100, 4, chunk_size=2), columns=list("abcd"))
-        print(df.execute())
-        assert await autoscaler_ref.get_dynamic_worker_nums() > 1
-        while await autoscaler_ref.get_dynamic_worker_nums() > 1:
-            dynamic_workers = await autoscaler_ref.get_dynamic_workers()
-            print(f"Waiting workers {dynamic_workers} to be released.")
-            await asyncio.sleep(1)
-        # Test data on node of released worker can still be fetched
-        pd_df = df.to_pandas()
-        groupby_sum_df = df.rechunk(40).groupby("a").sum()
-        print(groupby_sum_df.execute())
-        while await autoscaler_ref.get_dynamic_worker_nums() > 1:
-            dynamic_workers = await autoscaler_ref.get_dynamic_workers()
-            print(f"Waiting workers {dynamic_workers} to be released.")
-            await asyncio.sleep(1)
-        assert df.to_pandas().to_dict() == pd_df.to_dict()
-        assert (
-            groupby_sum_df.to_pandas().to_dict() == pd_df.groupby("a").sum().to_dict()
-        )
-
-
 @require_ray
 @pytest.mark.asyncio
-def test_init_metrics_on_ray(ray_start_regular, create_cluster):
+def test_init_metrics_on_ray(ray_start_regular_shared, create_cluster):
     client = create_cluster[0]
     assert client.session
     from ....metrics import api
diff --git a/mars/deploy/oscar/tests/test_ray_client.py b/mars/deploy/oscar/tests/test_ray_client.py
new file mode 100644
--- /dev/null
+++ b/mars/deploy/oscar/tests/test_ray_client.py
@@ -0,0 +1,77 @@
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import sys
+import tempfile
+import threading
+
+from .test_ray import new_ray_session_test
+from ....tests.core import require_ray
+from ....utils import lazy_import
+
+ray = lazy_import("ray")
+
+
+@require_ray
+def test_ray_client():
+    server_code = """import time
+import ray.util.client.server.server as ray_client_server
+
+server = ray_client_server.init_and_serve("{address}", num_cpus=20)
+print("OK", flush=True)
+while True:
+    time.sleep(1)
+"""
+
+    address = "127.0.0.1:50051"
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as f:
+        f.write(server_code.format(address=address))
+        f.flush()
+
+        proc = subprocess.Popen([sys.executable, "-u", f.name], stdout=subprocess.PIPE)
+
+        try:
+
+            def _check_ready(expect_exit=False):
+                while True:
+                    line = proc.stdout.readline()
+                    if proc.returncode is not None:
+                        if expect_exit:
+                            break
+                        raise Exception(
+                            f"Failed to start ray server at {address}, "
+                            f"the return code is {proc.returncode}."
+                        )
+                    if b"OK" in line:
+                        break
+
+            # Avoid ray.init timeout.
+            _check_ready()
+
+            # Avoid blocking the subprocess when the stdout pipe is full.
+            t = threading.Thread(target=_check_ready, args=(True,), daemon=True)
+            t.start()
+
+            ray.init(f"ray://{address}")
+            ray._inside_client_test = True
+            try:
+                new_ray_session_test()
+            finally:
+                ray._inside_client_test = False
+                ray.shutdown()
+        finally:
+            proc.kill()
+            proc.wait()
diff --git a/mars/deploy/oscar/tests/test_ray_dag.py b/mars/deploy/oscar/tests/test_ray_dag.py
--- a/mars/deploy/oscar/tests/test_ray_dag.py
+++ b/mars/deploy/oscar/tests/test_ray_dag.py
@@ -118,11 +118,12 @@ async def test_iterative_tiling(ray_start_regular_shared2, create_cluster):
 
 @require_ray
 @pytest.mark.parametrize("config", [{"backend": "ray"}])
-def test_sync_execute(config):
+def test_sync_execute(ray_start_regular_shared2, config):
     test_local.test_sync_execute(config)
 
 
 @require_ray
+@pytest.mark.skip("Enable when ray progress got fixed")
 @pytest.mark.asyncio
 async def test_session_get_progress(ray_start_regular_shared2, create_cluster):
     await test_local.test_session_get_progress(create_cluster)
@@ -136,7 +137,7 @@ def test_cancel(ray_start_regular_shared2, create_cluster, test_func):
 
 @require_ray
 @pytest.mark.parametrize("config", [{"backend": "ray"}])
-def test_executor_context_gc(config):
+def test_executor_context_gc(ray_start_regular_shared2, config):
     session = new_session(
         backend=config["backend"],
         n_cpu=2,
diff --git a/mars/deploy/oscar/tests/test_ray_fault_injection.py b/mars/deploy/oscar/tests/test_ray_fault_injection.py
--- a/mars/deploy/oscar/tests/test_ray_fault_injection.py
+++ b/mars/deploy/oscar/tests/test_ray_fault_injection.py
@@ -52,7 +52,6 @@ async def fault_cluster(request):
     ray_config.update(FAULT_INJECTION_CONFIG)
     ray_config.update(param.get("config", {}))
     client = await new_cluster(
-        "test_cluster",
         worker_num=2,
         worker_cpu=2,
         worker_mem=1 * 1024**3,
@@ -96,7 +95,7 @@ async def fault_cluster(request):
 )
 @pytest.mark.asyncio
 async def test_fault_inject_subtask_processor(
-    ray_start_regular, fault_cluster, fault_and_exception
+    ray_start_regular_shared, fault_cluster, fault_and_exception
 ):
     await test_fault_injection.test_fault_inject_subtask_processor(
         fault_cluster, fault_and_exception
@@ -128,7 +127,7 @@ async def test_fault_inject_subtask_processor(
     ],
 )
 @pytest.mark.asyncio
-async def test_rerun_subtask(ray_start_regular, fault_cluster, fault_config):
+async def test_rerun_subtask(ray_start_regular_shared, fault_cluster, fault_config):
     await test_fault_injection.test_rerun_subtask(fault_cluster, fault_config)
 
 
@@ -148,7 +147,9 @@ async def test_rerun_subtask(ray_start_regular, fault_cluster, fault_config):
     ],
 )
 @pytest.mark.asyncio
-async def test_rerun_subtask_describe(ray_start_regular, fault_cluster, fault_config):
+async def test_rerun_subtask_describe(
+    ray_start_regular_shared, fault_cluster, fault_config
+):
     await test_fault_injection.test_rerun_subtask_describe(fault_cluster, fault_config)
 
 
@@ -174,7 +175,9 @@ async def test_rerun_subtask_describe(ray_start_regular, fault_cluster, fault_co
     ],
 )
 @pytest.mark.asyncio
-async def test_rerun_subtask_fail(ray_start_regular, fault_cluster, fault_config):
+async def test_rerun_subtask_fail(
+    ray_start_regular_shared, fault_cluster, fault_config
+):
     await test_fault_injection.test_rerun_subtask_fail(fault_cluster, fault_config)
 
 
@@ -200,5 +203,5 @@ async def test_rerun_subtask_fail(ray_start_regular, fault_cluster, fault_config
     ],
 )
 @pytest.mark.asyncio
-async def test_retryable(ray_start_regular, fault_cluster, fault_config):
+async def test_retryable(ray_start_regular_shared, fault_cluster, fault_config):
     await test_fault_injection.test_retryable(fault_cluster, fault_config)
diff --git a/mars/deploy/oscar/tests/test_ray_scheduling.py b/mars/deploy/oscar/tests/test_ray_scheduling.py
--- a/mars/deploy/oscar/tests/test_ray_scheduling.py
+++ b/mars/deploy/oscar/tests/test_ray_scheduling.py
@@ -11,17 +11,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import asyncio
 
+import numpy as np
 import pytest
+import pandas as pd
+import time
 
+from .... import dataframe as md
+from .... import oscar as mo
+from .... import tensor as mt
+from ....oscar.backends.ray.utils import (
+    process_address_to_placement,
+    process_placement_to_address,
+    kill_and_wait,
+)
+from ....services.cluster import ClusterAPI
+from ....services.scheduling.supervisor.autoscale import AutoscalerActor
 from ....tests.core import require_ray
 from ....utils import lazy_import
-from ..ray import new_cluster
+from ..ray import new_cluster, _load_config
 from ..tests import test_local
-from .modules.utils import (  # noqa: F401; pylint: disable=unused-variable
-    cleanup_third_party_modules_output,
-    get_output_filenames,
-)
 
 ray = lazy_import("ray")
 
@@ -58,3 +68,210 @@ async def speculative_cluster():
 @pytest.mark.asyncio
 async def test_task_speculation_execution(ray_large_cluster, speculative_cluster):
     await test_local.test_task_speculation_execution(speculative_cluster)
+
+
+@pytest.mark.parametrize(
+    "ray_large_cluster", [{"num_nodes": 1, "num_cpus": 3}], indirect=True
+)
+@require_ray
+@pytest.mark.asyncio
+async def test_request_worker(ray_large_cluster):
+    worker_cpu, worker_mem = 1, 100 * 1024**2
+    client = await new_cluster(
+        worker_num=0, worker_cpu=worker_cpu, worker_mem=worker_mem
+    )
+    async with client:
+        cluster_state_ref = client._cluster._cluster_backend.get_cluster_state_ref()
+        # Note that supervisor took one node
+        workers = await asyncio.gather(
+            *[cluster_state_ref.request_worker(timeout=5) for _ in range(2)]
+        )
+        assert all(worker is not None for worker in workers)
+        assert not await cluster_state_ref.request_worker(timeout=5)
+        release_workers = [
+            cluster_state_ref.release_worker(worker) for worker in workers
+        ]
+        # Duplicate release workers requests should be handled.
+        release_workers.extend(
+            [cluster_state_ref.release_worker(worker) for worker in workers]
+        )
+        await asyncio.gather(*release_workers)
+        assert await cluster_state_ref.request_worker(timeout=5)
+        cluster_state_ref.reconstruct_worker()
+
+
+@pytest.mark.parametrize(
+    "ray_large_cluster", [{"num_nodes": 1, "num_cpus": 3}], indirect=True
+)
+@require_ray
+@pytest.mark.asyncio
+async def test_reconstruct_worker(ray_large_cluster):
+    worker_cpu, worker_mem = 1, 100 * 1024**2
+    client = await new_cluster(
+        worker_num=0, worker_cpu=worker_cpu, worker_mem=worker_mem
+    )
+    async with client:
+        cluster_api = await ClusterAPI.create(client._cluster.supervisor_address)
+        worker = await cluster_api.request_worker(timeout=5)
+        pg_name, bundle_index, process_index = process_address_to_placement(worker)
+        worker_sub_pool = process_placement_to_address(
+            pg_name, bundle_index, process_index + 1
+        )
+
+        worker_actor = ray.get_actor(worker)
+        worker_pid = await worker_actor.getpid.remote()
+        # the worker pool actor should be destroyed even we get actor.
+        worker_sub_pool_actor = ray.get_actor(worker_sub_pool)
+        worker_sub_pool_pid = await worker_sub_pool_actor.getpid.remote()
+
+        # kill worker main pool
+        await kill_and_wait(ray.get_actor(worker))
+
+        # duplicated reconstruct worker request can be handled.
+        await asyncio.gather(
+            cluster_api.reconstruct_worker(worker),
+            cluster_api.reconstruct_worker(worker),
+        )
+        worker_actor = ray.get_actor(worker)
+        new_worker_pid = await worker_actor.getpid.remote()
+        worker_sub_pool_actor = ray.get_actor(worker_sub_pool)
+        new_worker_sub_pool_pid = await worker_sub_pool_actor.getpid.remote()
+        assert new_worker_pid != worker_pid
+        assert new_worker_sub_pool_pid != worker_sub_pool_pid
+
+        # the compute should be ok after the worker is reconstructed.
+        raw = np.random.RandomState(0).rand(10, 5)
+        a = mt.tensor(raw, chunk_size=5).sum(axis=1)
+        b = a.execute(show_progress=False)
+        assert b is a
+        result = a.fetch()
+        np.testing.assert_array_equal(result, raw.sum(axis=1))
+
+
+@pytest.mark.parametrize(
+    "ray_large_cluster", [{"num_nodes": 2, "num_cpus": 4}], indirect=True
+)
+@pytest.mark.parametrize("init_workers", [0, 1])
+@require_ray
+@pytest.mark.asyncio
+async def test_auto_scale_out(ray_large_cluster, init_workers: int):
+    client = await new_cluster(
+        worker_num=init_workers,
+        worker_cpu=2,
+        worker_mem=200 * 1024**2,
+        supervisor_mem=1 * 1024**3,
+        config={
+            "scheduling.autoscale.enabled": True,
+            "scheduling.autoscale.scheduler_backlog_timeout": 1,
+            "scheduling.autoscale.worker_idle_timeout": 10000000,
+            "scheduling.autoscale.max_workers": 10,
+        },
+    )
+    async with client:
+
+        def time_consuming(x):
+            time.sleep(1)
+            return x * x
+
+        series_size = 100
+        assert (
+            md.Series(list(range(series_size)), chunk_size=1)
+            .apply(time_consuming)
+            .sum()
+            .execute()
+            .fetch()
+            == pd.Series(list(range(series_size))).apply(lambda x: x * x).sum()
+        )
+        autoscaler_ref = mo.create_actor_ref(
+            uid=AutoscalerActor.default_uid(),
+            address=client._cluster.supervisor_address,
+        )
+        assert await autoscaler_ref.get_dynamic_worker_nums() > 0
+
+
+@pytest.mark.timeout(timeout=600)
+@pytest.mark.parametrize(
+    "ray_large_cluster", [{"num_nodes": 2, "num_cpus": 4}], indirect=True
+)
+@require_ray
+@pytest.mark.asyncio
+async def test_auto_scale_in(ray_large_cluster):
+    config = _load_config()
+    config["scheduling"]["autoscale"]["enabled"] = True
+    config["scheduling"]["autoscale"]["worker_idle_timeout"] = 1
+    config["scheduling"]["autoscale"]["max_workers"] = 4
+    config["scheduling"]["autoscale"]["min_workers"] = 2
+    client = await new_cluster(
+        worker_num=0,
+        worker_cpu=2,
+        worker_mem=200 * 102**2,
+        supervisor_mem=1 * 1024**3,
+        config=config,
+    )
+    async with client:
+        autoscaler_ref = mo.create_actor_ref(
+            uid=AutoscalerActor.default_uid(),
+            address=client._cluster.supervisor_address,
+        )
+        new_worker_nums = 3
+        await asyncio.gather(
+            *[autoscaler_ref.request_worker() for _ in range(new_worker_nums)]
+        )
+        series_size = 100
+        assert (
+            md.Series(list(range(series_size)), chunk_size=20).sum().execute().fetch()
+            == pd.Series(list(range(series_size))).sum()
+        )
+        while await autoscaler_ref.get_dynamic_worker_nums() > 2:
+            dynamic_workers = await autoscaler_ref.get_dynamic_workers()
+            print(f"Waiting workers {dynamic_workers} to be released.")
+            await asyncio.sleep(1)
+        await asyncio.sleep(1)
+        assert await autoscaler_ref.get_dynamic_worker_nums() == 2
+
+
+@pytest.mark.skip("Enable it when ray ownership bug is fixed")
+@pytest.mark.timeout(timeout=200)
+@pytest.mark.parametrize("ray_large_cluster", [{"num_nodes": 4}], indirect=True)
+@require_ray
+@pytest.mark.asyncio
+async def test_ownership_when_scale_in(ray_large_cluster):
+    client = await new_cluster(
+        worker_num=0,
+        worker_cpu=2,
+        worker_mem=200 * 1024**2,
+        supervisor_mem=200 * 1024**2,
+        config={
+            "scheduling.autoscale.enabled": True,
+            "scheduling.autoscale.scheduler_check_interval": 1,
+            "scheduling.autoscale.scheduler_backlog_timeout": 1,
+            "scheduling.autoscale.worker_idle_timeout": 10,
+            "scheduling.autoscale.min_workers": 1,
+            "scheduling.autoscale.max_workers": 4,
+        },
+    )
+    async with client:
+        autoscaler_ref = mo.create_actor_ref(
+            uid=AutoscalerActor.default_uid(),
+            address=client._cluster.supervisor_address,
+        )
+        await asyncio.gather(*[autoscaler_ref.request_worker() for _ in range(2)])
+        df = md.DataFrame(mt.random.rand(100, 4, chunk_size=2), columns=list("abcd"))
+        print(df.execute())
+        assert await autoscaler_ref.get_dynamic_worker_nums() > 1
+        while await autoscaler_ref.get_dynamic_worker_nums() > 1:
+            dynamic_workers = await autoscaler_ref.get_dynamic_workers()
+            print(f"Waiting workers {dynamic_workers} to be released.")
+            await asyncio.sleep(1)
+        # Test data on node of released worker can still be fetched
+        pd_df = df.to_pandas()
+        groupby_sum_df = df.rechunk(40).groupby("a").sum()
+        print(groupby_sum_df.execute())
+        while await autoscaler_ref.get_dynamic_worker_nums() > 1:
+            dynamic_workers = await autoscaler_ref.get_dynamic_workers()
+            print(f"Waiting workers {dynamic_workers} to be released.")
+            await asyncio.sleep(1)
+        assert df.to_pandas().to_dict() == pd_df.to_dict()
+        assert (
+            groupby_sum_df.to_pandas().to_dict() == pd_df.groupby("a").sum().to_dict()
+        )