Repository: rucio/rucio
Issue: Necromancer doesn't handle properly exception

Motivation
----------
For some reason not understood yet, we can have some replicas having `lock_cnt!=0` but no rows in the locks table. The necromancer now crashes because of this. It must be able to handle it properly :
```
2018-11-21 14:16:49,533    22982    CRITICAL    Thread [1/1] : Traceback (most recent call last):
 File "/usr/lib/python2.7/site-packages/rucio/daemons/necromancer.py", line 85, in necromancer
   update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
 File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 351, in new_funct
   result = function(*args, **kwargs)
 File "/usr/lib/python2.7/site-packages/rucio/core/rule.py", line 1761, in update_rules_for_lost_replica
   raise RucioException('Problem with the locks')
RucioException: An unknown exception occurred.
```


Changed files:
lib/rucio/core/rule.py

diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
index 8573482aa..c51584aba 100644
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -16,7 +16,7 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2015
@@ -213,7 +213,7 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
                 if meta is not None:
                     try:
                         meta = json.dumps(meta)
-                    except:
+                    except Exception:
                         meta = None
 
                 new_rule = models.ReplicationRule(account=account,
@@ -482,7 +482,7 @@ def add_rules(dids, rules, session=None):
                         if rule.get('meta') is not None:
                             try:
                                 meta = json.dumps(rule.get('meta'))
-                            except:
+                            except Exception:
                                 meta = None
                         else:
                             meta = None
@@ -1730,9 +1730,9 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, session=Non
 
     datasets = []
     parent_dids = rucio.core.did.list_parent_dids(scope=scope, name=name, session=session)
-    for p in parent_dids:
-        if {'name': p['name'], 'scope': p['scope']} not in datasets:
-            datasets.append({'name': p['name'], 'scope': p['scope']})
+    for parent in parent_dids:
+        if {'name': parent['name'], 'scope': parent['scope']} not in datasets:
+            datasets.append({'name': parent['name'], 'scope': parent['scope']})
 
     for lock in locks:
         rule = session.query(models.ReplicationRule).filter(models.ReplicationRule.id == lock.rule_id).with_for_update(nowait=nowait).one()
@@ -1764,21 +1764,21 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, session=Non
 
         session.delete(lock)
 
-    if replica.lock_cnt == 0:
-        replica.tombstone = OBSOLETE
-        replica.state = ReplicaState.UNAVAILABLE
-        session.query(models.DataIdentifier).filter_by(scope=scope, name=name).update({'availability': DIDAvailability.LOST})
-        for dts in datasets:
-            logging.info('File %s:%s bad at site %s is completely lost from dataset %s:%s. Will be marked as LOST and detached', scope, name, rse, dts['scope'], dts['name'])
-            rucio.core.did.detach_dids(scope=dts['scope'], name=dts['name'], dids=[{'scope': scope, 'name': name}], session=session)
-            add_message('LOST', {'scope': scope,
-                                 'name': name,
-                                 'dataset_name': dts['name'],
-                                 'dataset_scope': dts['scope']},
-                        session=session)
-    else:
-        # This should never happen
-        raise RucioException('Problem with the locks')
+    if replica.lock_cnt != 0:
+        logging.error('Replica for did %s:%s with lock_cnt = %s. This should never happen. Update lock_cnt', scope, name, replica.lock_cnt)
+        replica.lock_cnt = 0
+
+    replica.tombstone = OBSOLETE
+    replica.state = ReplicaState.UNAVAILABLE
+    session.query(models.DataIdentifier).filter_by(scope=scope, name=name).update({'availability': DIDAvailability.LOST})
+    for dts in datasets:
+        logging.info('File %s:%s bad at site %s is completely lost from dataset %s:%s. Will be marked as LOST and detached', scope, name, rse, dts['scope'], dts['name'])
+        rucio.core.did.detach_dids(scope=dts['scope'], name=dts['name'], dids=[{'scope': scope, 'name': name}], session=session)
+        add_message('LOST', {'scope': scope,
+                             'name': name,
+                             'dataset_name': dts['name'],
+                             'dataset_scope': dts['scope']},
+                    session=session)
 
 
 @transactional_session
@@ -2943,7 +2943,7 @@ def __create_rule_approval_email(rule, session=None):
                         free_space_after = 'undefined'
                     else:
                         free_space_after = sizefmt(usage['free'] - did['bytes'])
-        except:
+        except Exception:
             pass
 
     # Resolve recipents:
@@ -2998,7 +2998,7 @@ def __create_recipents_list(rse_expression, session=None):
                     email = get_account(account=account, session=session).email
                     if email:
                         recipents.append((email, account))
-                except:
+                except Exception:
                     pass
 
     # LOCALGROUPDISK/LOCALGROUPTAPE
@@ -3013,7 +3013,7 @@ def __create_recipents_list(rse_expression, session=None):
                         email = get_account(account=account[0], session=session).email
                         if email:
                             recipents.append((email, account[0]))
-                    except:
+                    except Exception:
                         pass
 
     # GROUPDISK
@@ -3028,7 +3028,7 @@ def __create_recipents_list(rse_expression, session=None):
                         email = get_account(account=account[0], session=session).email
                         if email:
                             recipents.append((email, account[0]))
-                    except:
+                    except Exception:
                         pass
 
     # DDMADMIN as default