Repository: jupyterhub/jupyterhub

Issue:
Implementation details about the delete_forever hook optionally on a Spawner object
#3337 introduced a `delete_forever` hook for the Spawner base class and I want to deliberate around some details.

The only change besides a test case is the following lines of code.

https://github.com/jupyterhub/jupyterhub/blob/3fec19d191119e4203c243f44c43e74c8fe62f04/jupyterhub/apihandlers/users.py#L241-L245

## Question 1

Do we want the Spawner base class to implement a placeholder function that only have `pass` in it? It would become an entrypoint to write documentation

## Question 2

If not a yes on Question 1, do we want this check added? Now we will write an error message for all Spawners no matter if they have this functionality.

```diff
# allow the spawner to cleanup any persistent resources associated with the user 
try: 
+   if user.spawner.delete_forever:
        await user.spawner.delete_forever() 
    except Exception as e: 
        self.log.error("Error cleaning up persistent resources: %s" % e) 
```


Discussion:
> Do we want the Spawner base class to implement a placeholder function that only have pass in it? It would become an entrypoint to write documentation

I thought we had this! Oops! I missed in review that the `delete_forever` method addd in #3337 was defined in the tests, and not in the base class. It should definitely be in the base class. I'll do a PR cleaning up a bit of the logic that I see isn't quite correct.

Resolving patch:
diff --git a/jupyterhub/apihandlers/users.py b/jupyterhub/apihandlers/users.py
--- a/jupyterhub/apihandlers/users.py
+++ b/jupyterhub/apihandlers/users.py
@@ -242,11 +242,7 @@ async def delete(self, name):
 
         await maybe_future(self.authenticator.delete_user(user))
 
-        # allow the spawner to cleanup any persistent resources associated with the user
-        try:
-            await user.spawner.delete_forever()
-        except Exception as e:
-            self.log.error("Error cleaning up persistent resources: %s" % e)
+        await user.delete_spawners()
 
         # remove from registry
         self.users.delete(user)
@@ -488,10 +484,18 @@ async def delete(self, name, server_name=''):
         options = self.get_json_body()
         remove = (options or {}).get('remove', False)
 
-        def _remove_spawner(f=None):
-            if f and f.exception():
-                return
+        async def _remove_spawner(f=None):
+            """Remove the spawner object
+
+            only called after it stops successfully
+            """
+            if f:
+                # await f, stop on error,
+                # leaving resources in the db in case of failure to stop
+                await f
             self.log.info("Deleting spawner %s", spawner._log_name)
+            await maybe_future(user._delete_spawner(spawner))
+
             self.db.delete(spawner.orm_spawner)
             user.spawners.pop(server_name, None)
             self.db.commit()
@@ -512,7 +516,8 @@ def _remove_spawner(f=None):
             self.set_header('Content-Type', 'text/plain')
             self.set_status(202)
             if remove:
-                spawner._stop_future.add_done_callback(_remove_spawner)
+                # schedule remove when stop completes
+                asyncio.ensure_future(_remove_spawner(spawner._stop_future))
             return
 
         if spawner.pending:
@@ -530,9 +535,10 @@ def _remove_spawner(f=None):
 
         if remove:
             if stop_future:
-                stop_future.add_done_callback(_remove_spawner)
+                # schedule remove when stop completes
+                asyncio.ensure_future(_remove_spawner(spawner._stop_future))
             else:
-                _remove_spawner()
+                await _remove_spawner()
 
         status = 202 if spawner._stop_pending else 204
         self.set_header('Content-Type', 'text/plain')
diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py
--- a/jupyterhub/spawner.py
+++ b/jupyterhub/spawner.py
@@ -1141,6 +1141,18 @@ async def poll(self):
         """
         raise NotImplementedError("Override in subclass. Must be a coroutine.")
 
+    def delete_forever(self):
+        """Called when a user or server is deleted.
+
+        This can do things like request removal of resources such as persistent storage.
+        Only called on stopped spawners, and is usually the last action ever taken for the user.
+
+        Will only be called once on each Spawner, immediately prior to removal.
+
+        Stopping a server does *not* call this method.
+        """
+        pass
+
     def add_poll_callback(self, callback, *args, **kwargs):
         """Add a callback to fire when the single-user server stops"""
         if args or kwargs:
diff --git a/jupyterhub/user.py b/jupyterhub/user.py
--- a/jupyterhub/user.py
+++ b/jupyterhub/user.py
@@ -252,6 +252,35 @@ async def get_auth_state(self):
                 await self.save_auth_state(auth_state)
         return auth_state
 
+    async def delete_spawners(self):
+        """Call spawner cleanup methods
+
+        Allows the spawner to cleanup persistent resources
+        """
+        for name in self.orm_user.orm_spawners.keys():
+            await self._delete_spawner(name)
+
+    async def _delete_spawner(self, name_or_spawner):
+        """Delete a single spawner"""
+        # always ensure full Spawner
+        # this may instantiate the Spawner if it wasn't already running,
+        # just to delete it
+        if isinstance(name_or_spawner, str):
+            spawner = self.spawners[name_or_spawner]
+        else:
+            spawner = name_or_spawner
+
+        if spawner.active:
+            raise RuntimeError(
+                f"Spawner {spawner._log_name} is active and cannot be deleted."
+            )
+        try:
+            await maybe_future(spawner.delete_forever())
+        except Exception as e:
+            self.log.exception(
+                f"Error cleaning up persistent resources on {spawner._log_name}"
+            )
+
     def all_spawners(self, include_default=True):
         """Generator yielding all my spawners
 


Test patch:
diff --git a/jupyterhub/tests/mocking.py b/jupyterhub/tests/mocking.py
--- a/jupyterhub/tests/mocking.py
+++ b/jupyterhub/tests/mocking.py
@@ -93,16 +93,6 @@ def user_env(self, env):
     def _cmd_default(self):
         return [sys.executable, '-m', 'jupyterhub.tests.mocksu']
 
-    async def delete_forever(self):
-        """Called when a user is deleted.
-
-        This can do things like request removal of resources such as persistent storage.
-        Only called on stopped spawners, and is likely the last action ever taken for the user.
-
-        Will only be called once on the user's default Spawner.
-        """
-        pass
-
     use_this_api_token = None
 
     def start(self):