Repository: dmwm/wmcore
Issue: JobStatusLite should retry condor removal

Found some jobs pending in vocms0254 for 800k secs, thus beyond the default value for pending.
Looking at the logs, I see things like
```
2019-01-10 07:34:43,420:140328164853504:INFO:StatusPoller:Running job status poller algorithm...
2019-01-10 07:34:45,519:140328164853504:INFO:BossAirAPI:About to start building running jobs
2019-01-10 07:41:29,220:140328164853504:INFO:BossAirAPI:About to look for 40510 loadedJobs.
2019-01-10 07:41:31,352:140328164853504:INFO:BossAirAPI:Executing/changing/completing 40209/554/301 jobs in plugin SimpleCondorPlugin.
2019-01-10 07:41:31,352:140328164853504:INFO:BossAirAPI:About to change 554 jobs
2019-01-10 07:41:31,352:140328164853504:INFO:BossAirAPI:About to complete 301 jobs
2019-01-10 07:41:42,656:140328164853504:INFO:StatusPoller:Killing job 1 because it has exceeded timeout for status 'Pending'
...
2019-01-10 07:42:03,635:140328164853504:WARNING:SimpleCondorPlugin:Error while killing jobs on the schedd: ['3595.4', '3595.6', '3595.9',...
2019-01-10 07:42:04,483:140328164853504:INFO:BaseWorkerThread:StatusPoller took 441.064 secs to execute
```

where job_1 is:
3595.4  1  811044 /data/srv/wmagent/v1.1.18.patch2/install/wmagent/JobCreator/JobCache/prebello_Run2018B-v1-MuonEG-17Sep2018_1024p1_181005_214256_4615/DataProcessing/JobCollection_1_0/job_1/condor.3595.4.log T2_PL_Swierk


The expected behaviour would be:
1. condor_rm those jobs
2. check if condor fail to remove any of them,
3. if not, persist that info in the database
4. if it fail, then skip these jobs from any updates on the database

Changed files:
src/python/WMCore/BossAir/BossAirAPI.py, src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py, src/python/WMCore/BossAir/Plugins/MockPlugin.py, src/python/WMCore/BossAir/MySQL/UpdateJobs.py

diff --git a/src/python/WMCore/BossAir/BossAirAPI.py b/src/python/WMCore/BossAir/BossAirAPI.py
index 7d85c58fa..ae438fa47 100644
--- a/src/python/WMCore/BossAir/BossAirAPI.py
+++ b/src/python/WMCore/BossAir/BossAirAPI.py
@@ -615,9 +615,15 @@ class BossAirAPI(WMConnectionBase):
                 try:
                     pluginInst = self.plugins[plugin]
                     if workflowName:
+                        # jobs are completed regardless whether the kill succeeded or not
+                        self._completeKill(jobs=jobsToKill[plugin])
                         pluginInst.killWorkflowJobs(workflow=workflowName)
                     else:
-                        pluginInst.kill(jobs=jobsToKill[plugin])
+                        # raise an exception if it fails to kill jobs, such that the same
+                        # jobs are retried again in the next cycle
+                        pluginInst.kill(jobs=jobsToKill[plugin], raiseEx=True)
+                        self._completeKill(jobs=jobsToKill[plugin])
+
                     # Register the killed jobs
                     for job in jobsToKill[plugin]:
                         if job.get('cache_dir') is None or job.get('retry_count') is None:
@@ -644,6 +650,8 @@ class BossAirAPI(WMConnectionBase):
                             errorReport.save(filename=reportName)
                         except IOError as ioe:
                             logging.warning('Cannot write report %s because of %s', reportName, ioe)
+                except RuntimeError:
+                    logging.warning("Plugin failed to remove jobs. It will be retried in the next cycle.")
                 except WMException:
                     raise
                 except Exception as ex:
@@ -652,9 +660,6 @@ class BossAirAPI(WMConnectionBase):
                     logging.error(msg)
                     logging.debug("Interrupted while killing following jobs: %s\\n", jobsToKill[plugin])
                     raise BossAirException(msg)
-                finally:
-                    # Even if kill fails, complete the jobs
-                    self._completeKill(jobs=jobsToKill[plugin])
         return
 
     def update(self, jobs):
diff --git a/src/python/WMCore/BossAir/MySQL/UpdateJobs.py b/src/python/WMCore/BossAir/MySQL/UpdateJobs.py
index 0b59e920e..3ee2092aa 100644
--- a/src/python/WMCore/BossAir/MySQL/UpdateJobs.py
+++ b/src/python/WMCore/BossAir/MySQL/UpdateJobs.py
@@ -5,9 +5,9 @@ _UpdateJobs_
 MySQL implementation for updating jobs
 """
 
-
 from WMCore.Database.DBFormatter import DBFormatter
 
+
 class UpdateJobs(DBFormatter):
     """
     _UpdateJobs_
@@ -15,7 +15,6 @@ class UpdateJobs(DBFormatter):
     Update jobs with new values
     """
 
-
     sql = """UPDATE bl_runjob SET wmbs_id = :jobid, grid_id = :gridid,
                bulk_id = :bulkid, status_time = :status_time,
                sched_status = (SELECT id FROM bl_status WHERE name = :status),
@@ -24,9 +23,7 @@ class UpdateJobs(DBFormatter):
                WHERE id = :id
                """
 
-
-
-    def execute(self, jobs, conn = None, transaction = False):
+    def execute(self, jobs, conn=None, transaction=False):
         """
         _execute_
 
@@ -44,8 +41,6 @@ class UpdateJobs(DBFormatter):
                           'status_time': job.get('status_time', None), 'owner': job['userdn'],
                           'usergroup': job['usergroup'], 'userrole': job['userrole']})
 
-
-        result = self.dbi.processData(self.sql, binds, conn = conn,
-                                      transaction = transaction)
+        self.dbi.processData(self.sql, binds, conn=conn, transaction=transaction)
 
         return
diff --git a/src/python/WMCore/BossAir/Plugins/MockPlugin.py b/src/python/WMCore/BossAir/Plugins/MockPlugin.py
index b62c6e813..2cc162c80 100644
--- a/src/python/WMCore/BossAir/Plugins/MockPlugin.py
+++ b/src/python/WMCore/BossAir/Plugins/MockPlugin.py
@@ -196,7 +196,7 @@ class MockPlugin(BasePlugin):
 
         return runningList, changeList, completeList
 
-    def kill(self, jobs):
+    def kill(self, jobs, raiseEx=False):
         """
         _kill_
 
diff --git a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py
index 3fc0edf00..303967568 100644
--- a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py
+++ b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py
@@ -407,7 +407,7 @@ class SimpleCondorPlugin(BasePlugin):
 
         return jobtokill
 
-    def kill(self, jobs):
+    def kill(self, jobs, raiseEx=False):
         """
         _kill_
 
@@ -422,6 +422,8 @@ class SimpleCondorPlugin(BasePlugin):
             schedd.act(htcondor.JobAction.Remove, gridIds)
         except RuntimeError:
             logging.warn("Error while killing jobs on the schedd: %s", gridIds)
+            if raiseEx:
+                raise
 
         return