Repository: rucio/rucio
Issue: clients: download api, logger

Motivation
----------

STAGEIN FAILED: Get error: Staging input file failed: lfn=EVNT.04972714._000032.pool.root.1, error=stageIn with API faied: Logger instance has no attribute 'getChild'

Modification
------------

can be realted to:
https://github.com/dsoprea/GDriveFS/issues/104



Changed files:
lib/rucio/common/exception.py, lib/rucio/core/permission/atlas.py, lib/rucio/core/did.py, lib/rucio/core/rule.py, lib/rucio/common/policy.py, lib/rucio/client/uploadclient.py, lib/rucio/client/downloadclient.py

diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
index 78a67eb66..02d94c46e 100644
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -50,8 +50,8 @@ class DownloadClient:
         :param logger: logging.Logger object to use for downloads. If None nothing will be logged.
         """
         if not logger:
-            logger = logging.getLogger(__name__).getChild('null')
-            logger.addHandler(logging.NullHandler())
+            logger = logging.getLogger('%s.null' % __name__)
+            logger.disabled = True
 
         self.logger = logger
         self.is_human_readable = True
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
index 0d4fad6fe..5527ebf82 100644
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -49,8 +49,8 @@ class UploadClient:
         :param logger:      - logging.Logger object to use for uploads. If None nothing will be logged.
         """
         if not logger:
-            logger = logging.getLogger(__name__).getChild('null')
-            logger.addHandler(logging.NullHandler())
+            logger = logging.getLogger('%s.null' % __name__)
+            logger.disabled = True
 
         self.logger = logger
         self.client = _client if _client else Client()
diff --git a/lib/rucio/common/exception.py b/lib/rucio/common/exception.py
index 838cbeda4..93642013b 100644
--- a/lib/rucio/common/exception.py
+++ b/lib/rucio/common/exception.py
@@ -19,7 +19,7 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2017
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2012-2013
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2015
 # - Tobias Wegner <twegner@cern.ch>, 2018
 
@@ -859,3 +859,13 @@ class RSEChecksumUnavailable(RucioException):
         super(RSEChecksumUnavailable, self).__init__(*args, **kwargs)
         self._message = "RSE checksum unavailable."
         self.error_code = 81
+
+
+class UndefinedPolicy(RucioException):
+    """
+    Cannot find a defined policy in the Rucio config
+    """
+    def __init__(self, *args, **kwargs):
+        super(UndefinedPolicy, self).__init__(args, kwargs)
+        self._message = "No policy is defined."
+        self.error_code = 82
diff --git a/lib/rucio/common/policy.py b/lib/rucio/common/policy.py
index 86a37bfe0..1eee037f9 100644
--- a/lib/rucio/common/policy.py
+++ b/lib/rucio/common/policy.py
@@ -7,7 +7,7 @@
  http://www.apache.org/licenses/LICENSE-2.0
 
  Authors:
- - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2017
+ - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2018
  - Martin Barisits <martin.barisits@cern.ch>, 2017-2018
  - Vincent Garonne <vgaronne@gmail.com>, 2017-2018
  - Brian Bockelman <bbockelm@cse.unl.edu>, 2017
@@ -25,6 +25,7 @@ from dogpile.cache import make_region
 from dogpile.cache.api import NoValue
 
 from rucio.common.config import config_get
+from rucio.common.exception import UndefinedPolicy
 
 REGION = make_region().configure('dogpile.cache.memory',
                                  expiration_time=1800)
@@ -88,6 +89,6 @@ def policy_filter(function):
         return new_funct
 
     @wraps(function)
-    def null_funct(*args, **kwargs):
-        return
-    return null_funct
+    def raise_funct(*args, **kwargs):
+        raise UndefinedPolicy
+    return raise_funct
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
index 24496a1a0..eb13cae01 100644
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
@@ -404,13 +404,13 @@ def __add_collections_to_container(scope, name, collections, account, session):
     """
 
     condition = or_()
-    for c in collections:
+    for cond in collections:
 
-        if (scope == c['scope']) and (name == c['name']):
+        if (scope == cond['scope']) and (name == cond['name']):
             raise exception.UnsupportedOperation('Self-append is not valid!')
 
-        condition.append(and_(models.DataIdentifier.scope == c['scope'],
-                              models.DataIdentifier.name == c['name']))
+        condition.append(and_(models.DataIdentifier.scope == cond['scope'],
+                              models.DataIdentifier.name == cond['name']))
 
     available_dids = {}
     child_type = None
@@ -568,7 +568,10 @@ def delete_dids(dids, account, session=None):
 
         # ATLAS LOCALGROUPDISK Archive policy
         if did['did_type'] == DIDType.DATASET and did['scope'] != 'archive':
-            rucio.core.rule.archive_localgroupdisk_datasets(scope=did['scope'], name=did['name'], session=session)
+            try:
+                rucio.core.rule.archive_localgroupdisk_datasets(scope=did['scope'], name=did['name'], session=session)
+            except exception.UndefinedPolicy:
+                pass
 
         if did['purge_replicas'] is False:
             not_purge_replicas.append((did['scope'], did['name']))
diff --git a/lib/rucio/core/permission/atlas.py b/lib/rucio/core/permission/atlas.py
index bbbf93e4f..a13fddaee 100644
--- a/lib/rucio/core/permission/atlas.py
+++ b/lib/rucio/core/permission/atlas.py
@@ -382,14 +382,13 @@ def perm_attach_dids_to_dids(issuer, kwargs):
     """
     if issuer == 'root' or has_account_attribute(account=issuer, key='admin'):
         return True
-    else:
-        attachments = kwargs['attachments']
-        scopes = [did['scope'] for did in attachments]
-        scopes = list(set(scopes))
-        for scope in scopes:
-            if not rucio.core.scope.is_scope_owner(scope, issuer):
-                return False
-        return True
+    attachments = kwargs['attachments']
+    scopes = [did['scope'] for did in attachments]
+    scopes = list(set(scopes))
+    for scope in scopes:
+        if not rucio.core.scope.is_scope_owner(scope, issuer):
+            return False
+    return True
 
 
 def perm_create_did_sample(issuer, kwargs):
@@ -712,21 +711,21 @@ def perm_add_replicas(issuer, kwargs):
     :returns: True if account is allowed, otherwise False
     """
     rse = str(kwargs.get('rse', ''))
-    phys_group = []
+    group = []
 
     for kv in list_account_attributes(account=issuer):
-        if kv['key'].startswith('group-') and kv['value'] in ['admin', 'user']:
-            phys_group.append(kv['key'].partition('-')[2])
-    if phys_group:
-        rse_attr = list_rse_attributes(rse=rse)
+        if (kv['key'].startswith('group-') or kv['key'].startswith('country-')) and kv['value'] in ['admin', 'user']:
+            group.append(kv['key'].partition('-')[2])
+    rse_attr = list_rse_attributes(rse=rse)
+    if group:
         if rse_attr.get('type', '') == 'GROUPDISK':
-            if rse_attr.get('physgroup', '') in phys_group:
+            if rse_attr.get('physgroup', '') in group:
+                return True
+        if rse_attr.get('type', '') == 'LOCALGROUPDISK':
+            if rse_attr.get('country', '') in group:
                 return True
 
-    return rse.endswith('SCRATCHDISK')\\
-        or rse.endswith('USERDISK')\\
-        or rse.endswith('MOCK')\\
-        or rse.endswith('LOCALGROUPDISK')\\
+    return rse_attr.get('type', '') in ['SCRATCHDISK', 'MOCK', 'TEST']\\
         or issuer == 'root'\\
         or has_account_attribute(account=issuer, key='admin')
 
@@ -762,20 +761,23 @@ def perm_update_replicas_states(issuer, kwargs):
     :returns: True if account is allowed, otherwise False
     """
     rse = str(kwargs.get('rse', ''))
-    phys_group = []
+    group = []
 
     for kv in list_account_attributes(account=issuer):
-        if kv['key'].startswith('group-') and kv['value'] in ['admin', 'user']:
-            phys_group.append(kv['key'].partition('-')[2])
+        if (kv['key'].startswith('group-') or kv['key'].startswith('country-')) and kv['value'] in ['admin', 'user']:
+            group.append(kv['key'].partition('-')[2])
     rse_attr = list_rse_attributes(rse=rse)
-    if phys_group:
+    if group:
         if rse_attr.get('type', '') == 'GROUPDISK':
-            if rse_attr.get('physgroup', '') in phys_group:
+            if rse_attr.get('physgroup', '') in group:
+                return True
+        if rse_attr.get('type', '') == 'LOCALGROUPDISK':
+            if rse_attr.get('country', '') in group:
                 return True
-    else:
-        return rse_attr.get('type', '') in ['SCRATCHDISK', 'MOCK', 'LOCALGROUPDISK', 'TEST']\\
-            or issuer == 'root'\\
-            or has_account_attribute(account=issuer, key='admin')
+
+    return rse_attr.get('type', '') in ['SCRATCHDISK', 'MOCK', 'TEST']\\
+        or issuer == 'root'\\
+        or has_account_attribute(account=issuer, key='admin')
 
 
 def perm_queue_requests(issuer, kwargs):
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
index 8788db129..e7357dbd8 100644
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -16,7 +16,7 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2015
@@ -48,7 +48,7 @@ from rucio.common.exception import (InvalidRSEExpression, InvalidReplicationRule
                                     ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, RucioException,
                                     InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule,
                                     InvalidObject, RSEBlacklisted, RuleReplaceFailed, RequestNotFound,
-                                    ManualRuleApprovalBlocked, UnsupportedOperation)
+                                    ManualRuleApprovalBlocked, UnsupportedOperation, UndefinedPolicy)
 from rucio.common.schema import validate_schema
 from rucio.common.utils import str_to_date, sizefmt
 from rucio.core import account_counter, rse_counter, request as request_core
@@ -126,7 +126,10 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
                     raise StagingAreaRuleRequiresLifetime()
 
             # Check SCRATCHDISK Policy
-            lifetime = get_scratch_policy(account, rses, lifetime, session=session)
+            try:
+                lifetime = get_scratch_policy(account, rses, lifetime, session=session)
+            except UndefinedPolicy:
+                pass
 
             # Auto-lock rules for TAPE rses
             if not locked and lifetime is None:
@@ -374,7 +377,12 @@ def add_rules(dids, rules, session=None):
                             raise StagingAreaRuleRequiresLifetime()
 
                     # Check SCRATCHDISK Policy
-                    rule['lifetime'] = get_scratch_policy(rule.get('account'), rses, rule.get('lifetime', None), session=session)
+                    try:
+                        lifetime = get_scratch_policy(rule.get('account'), rses, rule.get('lifetime', None), session=session)
+                    except UndefinedPolicy:
+                        lifetime = rule.get('lifetime', None)
+
+                    rule['lifetime'] = lifetime
 
                     # 4.5 Get the lifetime
                     eol_at = define_eol(did.scope, did.name, rses, session=session)
@@ -1126,8 +1134,11 @@ def update_rule(rule_id, options, session=None):
             if key == 'lifetime':
                 # Check SCRATCHDISK Policy
                 rses = parse_expression(rule.rse_expression, session=session)
-                lifetime = get_scratch_policy(rule.account, rses, options['lifetime'], session=session)
-                rule.expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if options['lifetime'] is not None else None
+                try:
+                    lifetime = get_scratch_policy(rule.account, rses, options['lifetime'], session=session)
+                except UndefinedPolicy:
+                    lifetime = options['lifetime']
+                rule.expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if lifetime is not None else None
             if key == 'source_replica_expression':
                 rule.source_replica_expression = options['source_replica_expression']