Repository: rucio/rucio
Issue: policy_filter decorator makes methods fail in certain cases

Motivation
----------
When calling `lifetime = get_scratch_policy(account, rses, lifetime, session=session)` the `policy_filter` is applied to the method. However, if an organisation (such as generic) is set which is not in the mapping, the method is replaced by a null_funct, returning a NoneType. Thus in cases where the line above is called (with a lifetime of eg 100) it is replaced by `None`.

```
$ rucio add-rule user.mlassnig:my-test-dataset --lifetime 100 2 "MANCHESTER|CAMBRIDGE"
f69edfedf5de494397550640877a0974
$ rucio rule-info f69edfedf5de494397550640877a0974
...
Expires at:                 None
...
```

When doing update_rule with lifetime, the method even crashes completely, as it cannot handle the Nonetype in that case.

Expected behaviour
--------------

In cases where no respective method is set, the lifetime argument should just be passed through.

Changed files:
lib/rucio/common/exception.py, lib/rucio/core/did.py, lib/rucio/core/rule.py, lib/rucio/common/policy.py

diff --git a/lib/rucio/common/exception.py b/lib/rucio/common/exception.py
index 838cbeda4..93642013b 100644
--- a/lib/rucio/common/exception.py
+++ b/lib/rucio/common/exception.py
@@ -19,7 +19,7 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2017
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2012-2013
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2015
 # - Tobias Wegner <twegner@cern.ch>, 2018
 
@@ -859,3 +859,13 @@ class RSEChecksumUnavailable(RucioException):
         super(RSEChecksumUnavailable, self).__init__(*args, **kwargs)
         self._message = "RSE checksum unavailable."
         self.error_code = 81
+
+
+class UndefinedPolicy(RucioException):
+    """
+    Cannot find a defined policy in the Rucio config
+    """
+    def __init__(self, *args, **kwargs):
+        super(UndefinedPolicy, self).__init__(args, kwargs)
+        self._message = "No policy is defined."
+        self.error_code = 82
diff --git a/lib/rucio/common/policy.py b/lib/rucio/common/policy.py
index 86a37bfe0..1eee037f9 100644
--- a/lib/rucio/common/policy.py
+++ b/lib/rucio/common/policy.py
@@ -7,7 +7,7 @@
  http://www.apache.org/licenses/LICENSE-2.0
 
  Authors:
- - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2017
+ - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2018
  - Martin Barisits <martin.barisits@cern.ch>, 2017-2018
  - Vincent Garonne <vgaronne@gmail.com>, 2017-2018
  - Brian Bockelman <bbockelm@cse.unl.edu>, 2017
@@ -25,6 +25,7 @@ from dogpile.cache import make_region
 from dogpile.cache.api import NoValue
 
 from rucio.common.config import config_get
+from rucio.common.exception import UndefinedPolicy
 
 REGION = make_region().configure('dogpile.cache.memory',
                                  expiration_time=1800)
@@ -88,6 +89,6 @@ def policy_filter(function):
         return new_funct
 
     @wraps(function)
-    def null_funct(*args, **kwargs):
-        return
-    return null_funct
+    def raise_funct(*args, **kwargs):
+        raise UndefinedPolicy
+    return raise_funct
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
index 24496a1a0..eb13cae01 100644
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
@@ -404,13 +404,13 @@ def __add_collections_to_container(scope, name, collections, account, session):
     """
 
     condition = or_()
-    for c in collections:
+    for cond in collections:
 
-        if (scope == c['scope']) and (name == c['name']):
+        if (scope == cond['scope']) and (name == cond['name']):
             raise exception.UnsupportedOperation('Self-append is not valid!')
 
-        condition.append(and_(models.DataIdentifier.scope == c['scope'],
-                              models.DataIdentifier.name == c['name']))
+        condition.append(and_(models.DataIdentifier.scope == cond['scope'],
+                              models.DataIdentifier.name == cond['name']))
 
     available_dids = {}
     child_type = None
@@ -568,7 +568,10 @@ def delete_dids(dids, account, session=None):
 
         # ATLAS LOCALGROUPDISK Archive policy
         if did['did_type'] == DIDType.DATASET and did['scope'] != 'archive':
-            rucio.core.rule.archive_localgroupdisk_datasets(scope=did['scope'], name=did['name'], session=session)
+            try:
+                rucio.core.rule.archive_localgroupdisk_datasets(scope=did['scope'], name=did['name'], session=session)
+            except exception.UndefinedPolicy:
+                pass
 
         if did['purge_replicas'] is False:
             not_purge_replicas.append((did['scope'], did['name']))
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
index 8788db129..e7357dbd8 100644
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -16,7 +16,7 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2015
@@ -48,7 +48,7 @@ from rucio.common.exception import (InvalidRSEExpression, InvalidReplicationRule
                                     ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, RucioException,
                                     InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule,
                                     InvalidObject, RSEBlacklisted, RuleReplaceFailed, RequestNotFound,
-                                    ManualRuleApprovalBlocked, UnsupportedOperation)
+                                    ManualRuleApprovalBlocked, UnsupportedOperation, UndefinedPolicy)
 from rucio.common.schema import validate_schema
 from rucio.common.utils import str_to_date, sizefmt
 from rucio.core import account_counter, rse_counter, request as request_core
@@ -126,7 +126,10 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
                     raise StagingAreaRuleRequiresLifetime()
 
             # Check SCRATCHDISK Policy
-            lifetime = get_scratch_policy(account, rses, lifetime, session=session)
+            try:
+                lifetime = get_scratch_policy(account, rses, lifetime, session=session)
+            except UndefinedPolicy:
+                pass
 
             # Auto-lock rules for TAPE rses
             if not locked and lifetime is None:
@@ -374,7 +377,12 @@ def add_rules(dids, rules, session=None):
                             raise StagingAreaRuleRequiresLifetime()
 
                     # Check SCRATCHDISK Policy
-                    rule['lifetime'] = get_scratch_policy(rule.get('account'), rses, rule.get('lifetime', None), session=session)
+                    try:
+                        lifetime = get_scratch_policy(rule.get('account'), rses, rule.get('lifetime', None), session=session)
+                    except UndefinedPolicy:
+                        lifetime = rule.get('lifetime', None)
+
+                    rule['lifetime'] = lifetime
 
                     # 4.5 Get the lifetime
                     eol_at = define_eol(did.scope, did.name, rses, session=session)
@@ -1126,8 +1134,11 @@ def update_rule(rule_id, options, session=None):
             if key == 'lifetime':
                 # Check SCRATCHDISK Policy
                 rses = parse_expression(rule.rse_expression, session=session)
-                lifetime = get_scratch_policy(rule.account, rses, options['lifetime'], session=session)
-                rule.expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if options['lifetime'] is not None else None
+                try:
+                    lifetime = get_scratch_policy(rule.account, rses, options['lifetime'], session=session)
+                except UndefinedPolicy:
+                    lifetime = options['lifetime']
+                rule.expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if lifetime is not None else None
             if key == 'source_replica_expression':
                 rule.source_replica_expression = options['source_replica_expression']