Repository: dask/distributed

Issue:
Test failure in test_oversubscribing_leases
This showed up in a recent PR.  I copy-pasted the traceback and then restarted the job. 
 Unfortunately my attempt to copy-paste the trace turned out to be incomplete.  My apologies.  Hopefully this helps though.

cc @fjetter 

```python-traceback
>   assert not sem.acquire(timeout=0.05)
E   assert not True
E    +  where True = <bound method Semaphore.acquire of <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>>(timeout=0.05)
E    +    where <bound method Semaphore.acquire of <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>> = <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>.acquire

distributed\tests\test_semaphore.py:396: AssertionError
---------------------------- Captured stderr call -----------------------------
distributed.worker - WARNING -  Compute Failed
Function:  observe_state
args:      ()
kwargs:    {'sem': <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>}
Exception: AssertionError('assert not True\n +  where True = <bound method Semaphore.acquire of <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>>(timeout=0.05)\n +    where <bound method Semaphore.acquire of <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>> = <distributed.semaphore.Semaphore object at 0x0000027BF23DE438>.acquire',)

```


Discussion:
It's coming up here as well.  (I didn't restart the job this time)

https://github.com/dask/distributed/pull/3650/checks?check_run_id=593854022

Pinging also @lr4d 
Looking into it
Thanks!

On Fri, Apr 17, 2020 at 8:31 AM Florian Jetter <notifications@github.com>
wrote:

> Looking into it
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/3717#issuecomment-615311542>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACKZTHNTBLY3QM65WIHEDLRNBY6PANCNFSM4MKJ63HQ>
> .
>

I *think* that this is still happening.  I reran these tests about an hour ago, which I think is after the recent merge of #3720 

https://github.com/dask/distributed/pull/3650/checks?check_run_id=596031832

Resolving patch:
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -60,6 +60,7 @@ def __init__(self, scheduler):
                 "semaphore_release": self.release,
                 "semaphore_close": self.close,
                 "semaphore_refresh_leases": self.refresh_leases,
+                "semaphore_value": self.get_value,
             }
         )
 
@@ -79,6 +80,9 @@ def __init__(self, scheduler):
             dask.config.get("distributed.scheduler.locks.lease-timeout"), default="s",
         )
 
+    async def get_value(self, comm=None, name=None):
+        return len(self.leases[name])
+
     # `comm` here is required by the handler interface
     def create(self, comm=None, name=None, max_leases=None):
         # We use `self.max_leases` as the point of truth to find out if a semaphore with a specific
@@ -102,7 +106,7 @@ def refresh_leases(self, comm=None, name=None, lease_ids=None):
             for id_ in lease_ids:
                 if id_ not in self.leases[name]:
                     logger.critical(
-                        f"Trying to refresh an unknown lease ID {id_} for {name}. This might be due to leases "
+                        f"Refreshing an unknown lease ID {id_} for {name}. This might be due to leases "
                         f"timing out and may cause overbooking of the semaphore!"
                         f"This is often caused by long-running GIL-holding in the task which acquired the lease."
                     )
@@ -349,6 +353,7 @@ def __init__(self, max_leases=1, name=None, client=None):
         self._periodic_callback_name = f"refresh_semaphores_{self.id}"
         self.client._periodic_callbacks[self._periodic_callback_name] = pc
         pc.start()
+        self.refresh_leases = True
 
     def __await__(self):
         async def create_semaphore():
@@ -358,19 +363,16 @@ async def create_semaphore():
         return create_semaphore().__await__()
 
     async def _refresh_leases(self):
-        if self.client.scheduler is not None and not self._refreshing_leases:
-            self._refreshing_leases = True
-            if self._leases:
-                logger.debug(
-                    "%s refreshing leases for %s with IDs %s",
-                    self.client.id,
-                    self.name,
-                    self._leases,
-                )
-                await self.client.scheduler.semaphore_refresh_leases(
-                    lease_ids=list(self._leases), name=self.name
-                )
-            self._refreshing_leases = False
+        if self.refresh_leases and self._leases:
+            logger.debug(
+                "%s refreshing leases for %s with IDs %s",
+                self.client.id,
+                self.name,
+                self._leases,
+            )
+            await self.client.scheduler.semaphore_refresh_leases(
+                lease_ids=list(self._leases), name=self.name
+            )
 
     async def _acquire(self, timeout=None):
         lease_id = uuid.uuid4().hex
@@ -417,6 +419,12 @@ def release(self):
             self.client.scheduler.semaphore_release, name=self.name, lease_id=lease_id,
         )
 
+    def get_value(self):
+        """
+        Return the number of currently registered leases.
+        """
+        return self.client.sync(self.client.scheduler.semaphore_value, name=self.name)
+
     def __enter__(self):
         self.acquire()
         return self


Test patch:
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -2,7 +2,7 @@
 import dask
 import pytest
 from dask.distributed import Client
-
+from time import sleep
 from distributed import Semaphore
 from distributed.comm import Comm
 from distributed.core import ConnectionPool
@@ -10,6 +10,8 @@
 from distributed.utils_test import (  # noqa: F401
     client,
     cluster,
+    async_wait_for,
+    captured_logger,
     cluster_fixture,
     gen_cluster,
     slowidentity,
@@ -316,7 +318,7 @@ async def test_retry_acquire(c, s, a, b):
     client=True,
     config={
         "distributed.scheduler.locks.lease-timeout": "100ms",
-        "distributed.scheduler.locks.lease-validation-interval": "10ms",
+        "distributed.scheduler.locks.lease-validation-interval": "100ms",
     },
 )
 async def test_oversubscribing_leases(c, s, a, b):
@@ -346,6 +348,7 @@ async def test_oversubscribing_leases(c, s, a, b):
     # metadata check with a sleep loop is not elegant but practical.
     await c.set_metadata("release", False)
     sem = await Semaphore()
+    sem.refresh_callback.stop()
 
     def guaranteed_lease_timeout(x, sem):
         """
@@ -356,7 +359,7 @@ def guaranteed_lease_timeout(x, sem):
         all leases will eventually timeout. The function will only
         release/return once the "Event" is set, i.e. our observer is done.
         """
-        sem.refresh_callback.stop()
+        sem.refresh_leases = False
         client = get_client()
 
         with sem:
@@ -364,14 +367,17 @@ def guaranteed_lease_timeout(x, sem):
             # lease-timeout. This is twice the lease timeout to ensurre that the
             # leases are actually timed out
             slowidentity(delay=0.2)
-            old_value = client.set_metadata(x, "locked")
 
+            assert sem._leases
             # Now the GIL is free again, i.e. we enable the callback again
-            sem.refresh_callback.start()
+            sem.refresh_leases = True
+            sleep(0.1)
 
             # This is the poormans Event.wait()
-            while not client.get_metadata("release"):
-                slowidentity(delay=0.02)
+            while client.get_metadata("release") is not True:
+                sleep(0.05)
+
+            assert sem.get_value() >= 1
             return x
 
     def observe_state(sem):
@@ -382,22 +388,17 @@ def observe_state(sem):
         try to acquire and hopefully fail showing that the semaphore is
         protected if the oversubscription is recognized.
         """
-        client = get_client()
-        x_locked = False
-        y_locked = False
+        sem.refresh_callback.stop()
         # We wait until we're in an oversubscribed state, i.e. both tasks
         # are executed although there should only be one allowed
-        while not x_locked and y_locked:
-            slowidentity(delay=0.005)
-            x_locked = client.get_metadata(0) == "locked"
-            y_locked = client.get_metadata(1) == "locked"
-
-        # Once both are locked we should give the refresh time to notify the scheduler
-        # This parameter should be larger than ``lease-validation-interval``
-        slowidentity(delay=0.15)
+        while not sem.get_value() > 1:
+            sleep(0.2)
+
         # Once we're in an oversubscribed state, we must not be able to
         # acquire a lease.
         assert not sem.acquire(timeout=0)
+
+        client = get_client()
         client.set_metadata("release", True)
 
     observer = await Worker(s.address)
@@ -407,8 +408,18 @@ def observe_state(sem):
     )
     fut_observe = c.submit(observe_state, sem=sem, workers=[observer.address])
 
-    payload, observer = await c.gather([futures, fut_observe])
+    with captured_logger("distributed.semaphore") as caplog:
+        payload, observer = await c.gather([futures, fut_observe])
+
+    logs = caplog.getvalue().split("\n")
+    timeouts = [log for log in logs if "timed out" in log]
+    refresh_unknown = [log for log in logs if "Refreshing an unknown lease ID" in log]
+    assert len(timeouts) == 2
+    assert len(refresh_unknown) == 2
+
     assert sorted(payload) == [0, 1]
+    # Back to normal
+    assert await sem.get_value() == 0
 
 
 @gen_cluster(client=True,)
@@ -422,3 +433,15 @@ async def test_timeout_zero(c, s, a, b):
     assert await sem.acquire(timeout=0)
     assert not await sem.acquire(timeout=0)
     await sem.release()
+
+
+@gen_cluster(client=True)
+async def test_getvalue(c, s, a, b):
+
+    sem = await Semaphore()
+
+    assert await sem.get_value() == 0
+    await sem.acquire()
+    assert await sem.get_value() == 1
+    await sem.release()
+    assert await sem.get_value() == 0