Repository: VisTrails/tej (language: python)

Issue:
Provide way to clean up server
Provide method that removes the tej directory from the server. Right now there is no clean way of uninstalling tej if it is no longer in need. A workaround is `check_call('rm -rf ' + TEJ_DIR)` which is not very safe or logging on to the server, locating the directory, and deleting it.


Discussion / hints:
Example of work-around: https://github.com/JosuaKrause/parcell/blob/8d51c79040cbc3e878e37b04af0412ba3e91d982/parcell/loading.py#L626--L631
It's not clear to me why you'd want to do that. Most programs leave cache files behind, deleting (killing?) all the jobs should be enough. There is also a race condition removing the queue.

Fix patch:
diff --git a/tej/submission.py b/tej/submission.py
index c842f53..34124a3 100644
--- a/tej/submission.py
+++ b/tej/submission.py
@@ -292,7 +292,7 @@ def check_output(self, cmd):
         logger.debug("Output: %r", output)
         return output
 
-    def _resolve_queue(self, queue, depth=0):
+    def _resolve_queue(self, queue, depth=0, links=None):
         """Finds the location of tej's queue directory on the server.
 
         The `queue` set when constructing this `RemoteQueue` might be relative
@@ -344,6 +344,8 @@ def _resolve_queue(self, queue, depth=0):
         elif answer.startswith(b'tejdir: '):
             new = queue.parent / answer[8:]
             logger.debug("Found link to %s, recursing", new)
+            if links is not None:
+                links.append(queue)
             return self._resolve_queue(new, depth + 1)
         else:  # pragma: no cover
             logger.debug("Server returned %r", answer)
@@ -354,7 +356,8 @@ def _get_queue(self):
         """Gets the actual location of the queue, or None.
         """
         if self._queue is None:
-            queue, depth = self._resolve_queue(self.queue)
+            self._links = []
+            queue, depth = self._resolve_queue(self.queue, links=self._links)
             if queue is None and depth > 0:
                 raise QueueLinkBroken
             self._queue = queue
@@ -641,3 +644,27 @@ def list(self):
                 info = {}
         if job_id is not None:
             yield job_id, info
+
+    def cleanup(self, kill=False):
+        queue = self._get_queue()
+
+        if queue is not None:
+            # Kill jobs
+            for job_id, info in self.list():
+                if info['status'] == 'running':
+                    if not kill:
+                        raise JobStillRunning("Can't cleanup, some jobs are "
+                                              "still running")
+                    else:
+                        logger.info("Killing running job %s", job_id)
+                        self.kill(job_id)
+
+            # Remove queue
+            logger.info("Removing queue at %s", queue)
+            self.check_call('rm -rf -- %s' % shell_escape(queue))
+
+        # Remove links
+        for link in self._links:
+            self.check_call('rm -rf -- %s' % shell_escape(link))
+
+        return True


Test patch:
diff --git a/tests/functional.py b/tests/functional.py
index f8c3d20..77f5ee1 100644
--- a/tests/functional.py
+++ b/tests/functional.py
@@ -9,7 +9,7 @@
 import sys
 import time
 
-from tej.submission import make_unique_name
+from tej.submission import RemoteQueue, make_unique_name
 from tej.utils import unicode_
 
 
@@ -71,7 +71,8 @@ def functional_tests():
     logging.info("Creating default queue")
     check_call(tej + ['setup', destination])
     assert Path('~/.tej').expand_user().is_dir()
-    Path('~/.tej').expand_user().rmtree()
+    RemoteQueue(destination, '~/.tej').cleanup()
+    assert not Path('~/.tej').expand_user().exists()
 
     logging.info("Creating a queue with a link")
     check_call(tej + ['setup', destination,
@@ -214,3 +215,7 @@ def functional_tests():
     assert output == b''
     output = check_output(tej + ['status', destination, '--id', job_id])
     assert output == b'not found\n'
+
+    RemoteQueue(destination, 'tej 2/link').cleanup()
+    assert not Path('~/tej 2/link').expand_user().exists()
+    assert not Path('~/tej 2/queue').expand_user().exists()