Repository: mars-project/mars

Issue:
[BUG] `test_k_means.py::test_elkan_results` crash
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
`test_k_means.py::test_elkan_results` crash
**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version:  Python 3.8.5
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
```
root@af87dd8ccc01:/mars/mars# pytest -v -s mars/learn/cluster/tests/test_k_means.py::test_elkan_results[0.0001-normal]
=========================================================================== test session starts ===========================================================================
platform linux -- Python 3.8.5, pytest-7.0.1, pluggy-1.0.0 -- /mars/bin/python
cachedir: .pytest_cache
rootdir: /mars/mars, configfile: setup.cfg
plugins: asyncio-0.18.2, cov-3.0.0, forked-1.4.0, timeout-2.1.0
asyncio: mode=legacy
collected 1 item                                                                                                                                                          

mars/learn/cluster/tests/test_k_means.py::test_elkan_results[0.0001-normal] Fatal Python error: Bus error

Current thread 0x00007f5e8bfff700 (most recent call first):
  File "/mars/mars/mars/storage/core.py", line 111 in write
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 57 in run
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5eacffd700 (most recent call first):
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ee3212700 (most recent call first):
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5f06bec700 (most recent call first):
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5f023eb700 (most recent call first):
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ee8214700 (most recent call first):
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ebe7fc700 (most recent call first):
  File "/mars/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ebeffd700 (most recent call first):
  File "/mars/lib/python3.8/threading.py", line 302 in wait
  File "/mars/lib/python3.8/multiprocessing/queues.py", line 227 in _feed
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ebf7fe700 (most recent call first):
  File "/mars/lib/python3.8/threading.py", line 302 in wait
  File "/mars/lib/python3.8/multiprocessing/queues.py", line 227 in _feed
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ebffff700 (most recent call first):
  File "/mars/lib/python3.8/threading.py", line 302 in wait
  File "/mars/lib/python3.8/multiprocessing/queues.py", line 227 in _feed
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5ed69a0700 (most recent call first):
  File "/mars/mars/mars/lib/aio/isolation.py", line 36 in _run
  File "/mars/lib/python3.8/threading.py", line 870 in run
  File "/mars/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/mars/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f5f0bc73740 (most recent call first):
  File "/mars/lib/python3.8/threading.py", line 306 in wait
  File "/mars/lib/python3.8/concurrent/futures/_base.py", line 434 in result
  File "/mars/mars/mars/deploy/oscar/session.py", line 1601 in execute
  File "/mars/mars/mars/deploy/oscar/session.py", line 1803 in execute
  File "/mars/mars/mars/core/entity/executable.py", line 221 in execute
  File "/mars/mars/mars/learn/cluster/_k_means_common.py", line 378 in _relocate_empty_clusters
  File "/mars/mars/mars/learn/cluster/_k_means_elkan_iter.py", line 845 in elkan_iter
  File "/mars/mars/mars/learn/cluster/_kmeans.py", line 256 in _kmeans_single_elkan
  File "/mars/mars/mars/learn/cluster/_kmeans.py", line 925 in fit
  File "/mars/mars/mars/learn/cluster/tests/test_k_means.py", line 117 in test_elkan_results
  File "/mars/lib/python3.8/site-packages/_pytest/python.py", line 192 in pytest_pyfunc_call
  File "/mars/lib/python3.8/site-packages/pluggy/_callers.py", line 39 in _multicall
  File "/mars/lib/python3.8/site-packages/pluggy/_manager.py", line 80 in _hookexec
  File "/mars/lib/python3.8/site-packages/pluggy/_hooks.py", line 265 in __call__
  File "/mars/lib/python3.8/site-packages/_pytest/python.py", line 1718 in runtest
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 168 in pytest_runtest_call
  File "/mars/lib/python3.8/site-packages/pluggy/_callers.py", line 39 in _multicall
  File "/mars/lib/python3.8/site-packages/pluggy/_manager.py", line 80 in _hookexec
  File "/mars/lib/python3.8/site-packages/pluggy/_hooks.py", line 265 in __call__
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 261 in <lambda>
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 340 in from_call
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 260 in call_runtest_hook
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 221 in call_and_report
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 132 in runtestprotocol
  File "/mars/lib/python3.8/site-packages/_pytest/runner.py", line 113 in pytest_runtest_protocol
  File "/mars/lib/python3.8/site-packages/pluggy/_callers.py", line 39 in _multicall
  File "/mars/lib/python3.8/site-packages/pluggy/_manager.py", line 80 in _hookexec
  File "/mars/lib/python3.8/site-packages/pluggy/_hooks.py", line 265 in __call__
  File "/mars/lib/python3.8/site-packages/_pytest/main.py", line 347 in pytest_runtestloop
  File "/mars/lib/python3.8/site-packages/pluggy/_callers.py", line 39 in _multicall
  File "/mars/lib/python3.8/site-packages/pluggy/_manager.py", line 80 in _hookexec
  File "/mars/lib/python3.8/site-packages/pluggy/_hooks.py", line 265 in __call__
  File "/mars/lib/python3.8/site-packages/_pytest/main.py", line 322 in _main
  File "/mars/lib/python3.8/site-packages/_pytest/main.py", line 268 in wrap_session
  File "/mars/lib/python3.8/site-packages/_pytest/main.py", line 315 in pytest_cmdline_main
  File "/mars/lib/python3.8/site-packages/pluggy/_callers.py", line 39 in _multicall
  File "/mars/lib/python3.8/site-packages/pluggy/_manager.py", line 80 in _hookexec
  File "/mars/lib/python3.8/site-packages/pluggy/_hooks.py", line 265 in __call__
  File "/mars/lib/python3.8/site-packages/_pytest/config/__init__.py", line 165 in main
  File "/mars/lib/python3.8/site-packages/_pytest/config/__init__.py", line 188 in console_main
  File "/mars/bin/pytest", line 8 in <module>
Bus error

```



Discussion:

Resolving patch:
diff --git a/mars/services/lifecycle/supervisor/tracker.py b/mars/services/lifecycle/supervisor/tracker.py
--- a/mars/services/lifecycle/supervisor/tracker.py
+++ b/mars/services/lifecycle/supervisor/tracker.py
@@ -87,7 +87,10 @@ def _get_remove_chunk_keys(self, chunk_keys: List[str]):
         return to_remove_chunk_keys
 
     async def decref_chunks(self, chunk_keys: List[str]):
-        logger.debug("Decrease reference count for chunks %s", chunk_keys)
+        logger.debug(
+            "Decrease reference count for chunks %s",
+            {ck: self._chunk_ref_counts[ck] for ck in chunk_keys},
+        )
         to_remove_chunk_keys = self._get_remove_chunk_keys(chunk_keys)
         # make _remove_chunks release actor lock so that multiple `decref_chunks` can run concurrently.
         yield self._remove_chunks(to_remove_chunk_keys)
@@ -164,8 +167,14 @@ def incref_tileables(self, tileable_keys: List[str]):
                     f"tileable {tileable_key} " f"not tracked before"
                 )
             self._tileable_ref_counts[tileable_key] += 1
+            incref_chunk_keys = self._tileable_key_to_chunk_keys[tileable_key]
             # incref chunks for this tileable
-            self.incref_chunks(self._tileable_key_to_chunk_keys[tileable_key])
+            logger.debug(
+                "Incref chunks %s while decrefing tileable %s",
+                incref_chunk_keys,
+                tileable_key,
+            )
+            self.incref_chunks(incref_chunk_keys)
 
     async def decref_tileables(self, tileable_keys: List[str]):
         decref_chunk_keys = []
diff --git a/mars/services/storage/handler.py b/mars/services/storage/handler.py
--- a/mars/services/storage/handler.py
+++ b/mars/services/storage/handler.py
@@ -39,10 +39,10 @@
 logger = logging.getLogger(__name__)
 
 
-class StorageHandlerActor(mo.StatelessActor):
+class StorageHandlerActor(mo.Actor):
     """
     Storage handler actor, provide methods like `get`, `put`, etc.
-    This actor is stateless and created on worker's sub pools.
+    This actor is stateful and created on worker's sub pools.
     """
 
     def __init__(
diff --git a/mars/services/task/supervisor/processor.py b/mars/services/task/supervisor/processor.py
--- a/mars/services/task/supervisor/processor.py
+++ b/mars/services/task/supervisor/processor.py
@@ -276,6 +276,11 @@ def _get_decref_stage_chunk_keys(
     @_record_error
     async def decref_stage(self, stage_processor: "TaskStageProcessor"):
         decref_chunk_keys = self._get_decref_stage_chunk_keys(stage_processor)
+        logger.debug(
+            "Decref chunks %s when stage %s finish",
+            decref_chunk_keys,
+            stage_processor.stage_id,
+        )
         await self._lifecycle_api.decref_chunks(decref_chunk_keys)
 
     @decref_stage.batch
@@ -284,6 +289,7 @@ async def decref_stage(self, args_list, kwargs_list):
         decref_chunk_keys = []
         for args, kwargs in zip(args_list, kwargs_list):
             decref_chunk_keys.extend(self._get_decref_stage_chunk_keys(*args, **kwargs))
+        logger.debug("Decref chunks %s when stage finish", decref_chunk_keys)
         await self._lifecycle_api.decref_chunks(decref_chunk_keys)
 
     async def _get_next_chunk_graph(
@@ -897,6 +903,11 @@ async def _decref_input_subtasks(
                             # decref main key as well
                             decref_chunk_keys.extend([key[0] for key in data_keys])
                 decref_chunk_keys.append(result_chunk.key)
+        logger.debug(
+            "Decref chunks %s when subtask %s finish",
+            decref_chunk_keys,
+            subtask.subtask_id,
+        )
         await self._lifecycle_api.decref_chunks(decref_chunk_keys)
 
         # `set_subtask_result` will be called when subtask finished


Test patch:
diff --git a/mars/core/graph/tests/test_graph.py b/mars/core/graph/tests/test_graph.py
--- a/mars/core/graph/tests/test_graph.py
+++ b/mars/core/graph/tests/test_graph.py
@@ -15,6 +15,7 @@
 import pytest
 
 from .... import tensor as mt
+from ....tests.core import flaky
 from .. import DAG, GraphContainsCycleError
 
 
@@ -94,6 +95,7 @@ def test_dag():
         )
 
 
+@flaky(max_runs=3)
 def test_to_dot():
     arr = mt.random.randint(10, size=(10, 8), chunk_size=4)
     arr_add = mt.random.randint(10, size=(10, 8), chunk_size=4)
diff --git a/mars/dataframe/datastore/tests/test_datastore_execution.py b/mars/dataframe/datastore/tests/test_datastore_execution.py
--- a/mars/dataframe/datastore/tests/test_datastore_execution.py
+++ b/mars/dataframe/datastore/tests/test_datastore_execution.py
@@ -41,6 +41,7 @@
     vineyard = None
 
 from .... import dataframe as md
+from ....tests.core import flaky
 from ... import DataFrame
 
 
@@ -165,6 +166,7 @@ def test_to_sql():
 
 
 @pytest.mark.skipif(pa is None, reason="pyarrow not installed")
+@flaky(max_runs=3)
 def test_to_parquet_arrow_execution(setup):
     raw = pd.DataFrame(
         {