Repository: dask/distributed

Issue:
Proxy to worker dashboard not working


```
dask scheduler
dask worker <scheduler addr> --dashboard
```

Then navigate to `http://127.0.0.1:8787/info/main/workers.html` and click on dashboard. This should proxy to the worker dashboard but instead it triggers the below exception (with a 500 error code, of course)

```python-traceback
/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py:607: UserWarning: The Tornado web application does not have an 'identity_provider' defined in its settings. In future releases of jupyter_server, this will be a required key for all subclasses of `JupyterHandler`. For an example, see the jupyter_server source code for how to add an identity provider to the tornado settings: https://github.com/jupyter-server/jupyter_server/blob/v2.0.0/jupyter_server/serverapp.py#L242
  if type(self.identity_provider) is IdentityProvider and mod_obj.__name__ != __name__:
2023-11-10 14:28:01,113 - tornado.application - ERROR - Uncaught exception GET /proxy/50121/192.168.2.53/status (127.0.0.1)
HTTPServerRequest(protocol='http', host='127.0.0.1:8787', method='GET', uri='/proxy/50121/192.168.2.53/status', version='HTTP/1.1', remote_ip='127.0.0.1')
Traceback (most recent call last):
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 1765, in _execute
    result = await result  # type: ignore
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 622, in prepare
    _user = await _user
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/auth/identity.py", line 240, in _get_user
    _cookie_user = self.get_user_cookie(handler)
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/auth/identity.py", line 398, in get_user_cookie
    _user_cookie = handler.get_secure_cookie(
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 836, in get_signed_cookie
    self.require_setting("cookie_secret", "secure cookies")
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 1669, in require_setting
    raise Exception(
Exception: You must define the 'cookie_secret' setting in your application to use secure cookies
2023-11-10 14:28:01,121 - tornado.application - ERROR - Uncaught exception in write_error
Traceback (most recent call last):
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 1765, in _execute
    result = await result  # type: ignore
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 622, in prepare
    _user = await _user
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/auth/identity.py", line 240, in _get_user
    _cookie_user = self.get_user_cookie(handler)
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/auth/identity.py", line 398, in get_user_cookie
    _user_cookie = handler.get_secure_cookie(
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 836, in get_signed_cookie
    self.require_setting("cookie_secret", "secure cookies")
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 1669, in require_setting
    raise Exception(
Exception: You must define the 'cookie_secret' setting in your application to use secure cookies

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 1294, in send_error
    self.write_error(status_code, **kwargs)
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 719, in write_error
    html = self.render_template("%s.html" % status_code, **ns)
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 645, in render_template
    ns.update(self.template_namespace)
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 655, in template_namespace
    logged_in=self.logged_in,
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 197, in logged_in
    user = self.current_user
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/tornado/web.py", line 1420, in current_user
    self._current_user = self.get_current_user()
  File "/Users/fjetter/miniforge3/envs/coiled-benchmarks-310/lib/python3.10/site-packages/jupyter_server/base/handlers.py", line 176, in get_current_user
    raise RuntimeError(msg)
RuntimeError: Calling `GlobalProxyHandler.get_current_user()` directly is deprecated in jupyter-server 2.0. Use `self.current_user` instead (works in all versions).
```


```
jupyter_server            2.10.0             pyhd8ed1ab_0    conda-forge
jupyter_server_terminals  0.4.4              pyhd8ed1ab_1    conda-forge
jupyter-server-proxy      4.1.0              pyhd8ed1ab_0    conda-forge
tornado                   6.3.3           py310h2aa6e3c_1    conda-forge
```


Discussion:
FWIW setting a cookie secret just triggers another exception a little later on

Resolving patch:
diff --git a/distributed/http/proxy.py b/distributed/http/proxy.py
--- a/distributed/http/proxy.py
+++ b/distributed/http/proxy.py
@@ -19,6 +19,15 @@ def initialize(self, dask_server=None, extra=None):
             self.scheduler = dask_server
             self.extra = extra or {}
 
+        # `get_current_user` and `prepare` method impls reference
+        # issue in tornado & jupyter server compat here
+        # https://github.com/jupyter-server/jupyter_server/issues/1012
+        def get_current_user(self):
+            return "dask"
+
+        async def prepare(self):
+            web.authenticated(lambda rq: None)(self)
+
         async def http_get(self, port, host, proxied_path):
             # route here first
             # incoming URI /proxy/{port}/{host}/{proxied_path}
@@ -29,6 +38,9 @@ async def http_get(self, port, host, proxied_path):
             uri = f"/proxy/{port}/{proxied_path}"
             self.request.uri = uri
 
+            if self.host not in self.host_allowlist:
+                self.host_allowlist.append(self.host)
+
             # slash is removed during regex in handler
             proxied_path = "/%s" % proxied_path
 
@@ -41,6 +53,8 @@ async def http_get(self, port, host, proxied_path):
             return await self.proxy(port, proxied_path)
 
         async def open(self, port, host, proxied_path):
+            if host not in self.host_allowlist:
+                self.host_allowlist.append(host)
             # finally, proxy to other address/port
             return await self.proxy_open(host, port, proxied_path)
 


Test patch:
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -18,7 +18,7 @@
 from distributed.compatibility import LINUX, WINDOWS
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
-from distributed.utils import open_port
+from distributed.utils import get_ip, open_port
 from distributed.utils_test import (
     gen_cluster,
     inc,
@@ -485,39 +485,32 @@ def func(dask_worker):
 
 
 @pytest.mark.slow
-@gen_cluster(
-    client=True, nthreads=[], scheduler_kwargs={"dashboard_address": "localhost:8787"}
-)
-async def test_dashboard_non_standard_ports(c, s, requires_default_ports):
+def test_dashboard_non_standard_ports():
     pytest.importorskip("bokeh")
     requests = pytest.importorskip("requests")
 
-    try:
-        import jupyter_server_proxy  # noqa: F401
-
-        proxy_exists = True
-    except ImportError:
-        proxy_exists = False
-
-    with popen(
-        [
-            "dask",
-            "worker",
-            s.address,
-            "--dashboard-address",
-            ":4833",
-            "--host",
-            "127.0.0.1",
-        ]
-    ):
-        await c.wait_for_workers(1)
-
-        response = requests.get("http://127.0.0.1:4833/status")
+    s_host = "127.0.0.1"
+    # use internal ip instead of localhost ip to verify GlobalProxyHandler will update
+    # to allow internal host ip of a worker.
+    w_host = get_ip()
+    s_port = "3233"
+    s_dashboard_port = "3232"
+    w_dashboard_port = "4833"
+    s_cmd = f"dask scheduler --host {s_host} --port {s_port} --dashboard-address :{s_dashboard_port}"
+    w_cmd = f"dask worker {s_host}:{s_port} --dashboard-address :{w_dashboard_port} --host {w_host}"
+
+    with popen(s_cmd.split()), popen(w_cmd.split()):
+        with Client(f"{s_host}:{s_port}") as c:
+            c.wait_for_workers(1)
+
+        response = requests.get(f"http://{s_host}:{w_dashboard_port}/status")
         response.raise_for_status()
+
         # TEST PROXYING WORKS
-        if proxy_exists:
-            response = requests.get("http://127.0.0.1:8787/proxy/4833/127.0.0.1/status")
-            response.raise_for_status()
+        response = requests.get(
+            f"http://{s_host}:{s_dashboard_port}/proxy/{w_dashboard_port}/{w_host}/status"
+        )
+        response.raise_for_status()
 
     with pytest.raises(requests.ConnectionError):
         requests.get("http://localhost:4833/status/")
diff --git a/distributed/tests/test_jupyter.py b/distributed/tests/test_jupyter.py
--- a/distributed/tests/test_jupyter.py
+++ b/distributed/tests/test_jupyter.py
@@ -10,6 +10,7 @@
 from tornado.httpclient import AsyncHTTPClient
 
 from distributed import Client, Scheduler
+from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import Status
 from distributed.utils import open_port
 from distributed.utils_test import gen_test, popen
@@ -21,6 +22,17 @@
 
 pytestmark = pytest.mark.filterwarnings("ignore:Jupyter is migrating its paths")
 
+if WINDOWS:
+    try:
+        import jupyter_server  # noqa: F401
+    except ImportError:
+        pass
+    else:
+        pytest.skip(
+            allow_module_level=True,
+            reason="Windows struggles running these tests w/ jupyter server",
+        )
+
 
 @gen_test()
 async def test_jupyter_server():
@@ -73,7 +85,8 @@ async def test_jupyter_idle_timeout():
 
             assert s.status not in (Status.closed, Status.closing)
 
-        await asyncio.sleep(s.idle_timeout)
+        # small bit of extra time to catch up
+        await asyncio.sleep(s.idle_timeout + 0.5)
         assert s.status in (Status.closed, Status.closing)
 
 
@@ -94,12 +107,12 @@ async def test_jupyter_idle_timeout_returned():
         assert next_idle is not None
         assert next_idle > last_idle
 
-        assert s.check_idle() is None
-        # ^ NOTE: this probably should be `== next_idle`;
-        # see discussion in https://github.com/dask/distributed/pull/7687#discussion_r1145095196
+        assert s.check_idle() is next_idle
 
 
 @pytest.mark.slow
+@pytest.mark.xfail(WINDOWS, reason="Subprocess launching scheduler TimeoutError")
+@pytest.mark.xfail(MACOS, reason="Client fails to connect on OSX")
 def test_shutsdown_cleanly(loop):
     port = open_port()
     with concurrent.futures.ThreadPoolExecutor() as tpe: