Repository: rucio/rucio

Issue:
Xrd: Tiny fixes
Motivation
----------

- Uniform handling of authentication

- Fix for direct file upload paths


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/constraints.py b/lib/rucio/common/constraints.py
--- a/lib/rucio/common/constraints.py
+++ b/lib/rucio/common/constraints.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,11 +16,12 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
 from six import string_types
 
 # List of authorized value types for key
-AUTHORIZED_VALUE_TYPES = (float, int, string_types)
+AUTHORIZED_VALUE_TYPES = (float, int) + string_types
 STRING_TYPES = string_types
diff --git a/lib/rucio/daemons/conveyor/receiver.py b/lib/rucio/daemons/conveyor/receiver.py
--- a/lib/rucio/daemons/conveyor/receiver.py
+++ b/lib/rucio/daemons/conveyor/receiver.py
@@ -185,7 +185,7 @@ def receiver(id, total_threads=1, full_mode=False):
 
     logging.info('receiver starting in full mode: %s' % full_mode)
 
-    executable = ' '.join(sys.argv)
+    executable = 'conveyor-receiver'
     hostname = socket.getfqdn()
     pid = os.getpid()
     hb_thread = threading.current_thread()
diff --git a/lib/rucio/rse/protocols/xrootd.py b/lib/rucio/rse/protocols/xrootd.py
--- a/lib/rucio/rse/protocols/xrootd.py
+++ b/lib/rucio/rse/protocols/xrootd.py
@@ -1,21 +1,32 @@
-'''
-  Copyright European Organization for Nuclear Research (CERN)
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  You may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Authors:
-  - WeiJen Chang, <wchang@cern.ch>, 2013
-  - Cheng-Hsi Chao, <cheng-hsi.chao@cern.ch>, 2014
-  - Mario Lassnig, <mario.lassnig@cern.ch>, 2016-2017
-  - Nicolo Magini, <nicolo.magini@cern.ch>, 2018
-
-  PY3K COMPATIBLE
-'''
+# Copyright 2013-2020 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - WeiJen Chang <e4523744@gmail.com>, 2013
+# - Ralph Vigne <ralph.vigne@cern.ch>, 2013
+# - Cheng-Hsi Chao <cheng-hsi.chao@cern.ch>, 2014
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2016-2020
+# - Nicolo Magini <Nicolo.Magini@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Tomas Javurek <tomasjavurek09@gmail.com>, 2020
+# - Martin Barisits <martin.barisits@cern.ch>, 2020
+#
+# PY3K COMPATIBLE
 
 import os
+import logging
 
 from rucio.common import exception
 from rucio.rse.protocols import protocol
@@ -31,9 +42,15 @@ def __init__(self, protocol_attr, rse_settings, logger=None):
             :param props Properties derived from the RSE Repository
         """
         super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
+
+        if not logger:
+            logger = logging.getLogger('%s.null' % __name__)
+            logger.disabled = True
+
         self.scheme = self.attributes['scheme']
         self.hostname = self.attributes['hostname']
         self.port = str(self.attributes['port'])
+        self.logger = logger
 
     def path2pfn(self, path):
         """
@@ -44,6 +61,7 @@ def path2pfn(self, path):
             :returns: Fully qualified PFN.
 
         """
+        self.logger.debug('xrootd.path2pfn: path: {}'.format(path))
         if not path.startswith('xroot') and not path.startswith('root'):
             if path.startswith('/'):
                 return '%s://%s:%s/%s' % (self.scheme, self.hostname, self.port, path)
@@ -61,11 +79,13 @@ def exists(self, pfn):
 
             :raise  ServiceUnavailable
         """
+        self.logger.debug('xrootd.exists: pfn: {}'.format(pfn))
         try:
             path = self.pfn2path(pfn)
-            cmd = 'xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            self.logger.info('xrootd.exists: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            if not status == 0:
+            if status != 0:
                 return False
         except Exception as e:
             raise exception.ServiceUnavailable(e)
@@ -82,6 +102,7 @@ def stat(self, path):
 
         :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
         """
+        self.logger.debug('xrootd.stat: path: {}'.format(path))
         ret = {}
         chsum = None
         if path.startswith('root:'):
@@ -89,13 +110,15 @@ def stat(self, path):
 
         try:
             # xrdfs stat for getting filesize
-            cmd = 'xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            self.logger.info('xrootd.stat: filesize cmd: {}'.format(cmd))
             status_stat, out, err = execute(cmd)
             if status_stat == 0:
                 ret['filesize'] = out.split('\n')[2].split()[-1]
 
             # xrdfs query checksum for getting checksum
-            cmd = 'xrdfs %s:%s query checksum %s' % (self.hostname, self.port, path)
+            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s query checksum %s' % (self.hostname, self.port, path)
+            self.logger.info('xrootd.stat: checksum cmd: {}'.format(cmd))
             status_query, out, err = execute(cmd)
             if status_query == 0:
                 chsum, value = out.strip('\n').split()
@@ -120,6 +143,7 @@ def pfn2path(self, pfn):
 
         :returns: path.
         """
+        self.logger.debug('xrootd.pfn2path: pfn: {}'.format(pfn))
         if pfn.startswith('//'):
             return pfn
         elif pfn.startswith('/'):
@@ -138,6 +162,7 @@ def lfns2pfns(self, lfns):
 
         :returns: Fully qualified PFN.
         """
+        self.logger.debug('xrootd.lfns2pfns: lfns: {}'.format(lfns))
         pfns = {}
         prefix = self.attributes['prefix']
 
@@ -165,12 +190,14 @@ def connect(self):
 
             :raises RSEAccessDenied
         """
+        self.logger.debug('xrootd.connect: port: {}, hostname {}'.format(self.port, self.hostname))
         try:
             # The query stats call is not implemented on some xroot doors.
             # Workaround: fail, if server does not reply within 10 seconds for static config query
             cmd = 'XrdSecPROTOCOL=gsi XRD_REQUESTTIMEOUT=10 xrdfs %s:%s query config %s:%s' % (self.hostname, self.port, self.hostname, self.port)
+            self.logger.info('xrootd.connect: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            if not status == 0:
+            if status != 0:
                 raise exception.RSEAccessDenied(err)
         except Exception as e:
             raise exception.RSEAccessDenied(e)
@@ -188,9 +215,10 @@ def get(self, pfn, dest, transfer_timeout=None):
 
             :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
         """
-
+        self.logger.debug('xrootd.get: pfn: {}'.format(pfn))
         try:
             cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
+            self.logger.info('xrootd.get: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status == 54:
                 raise exception.SourceNotFound()
@@ -212,14 +240,18 @@ def put(self, filename, target, source_dir, transfer_timeout=None):
             :raises ServiceUnavailable: if some generic error occured in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
+        self.logger.debug('xrootd.put: filename: {} target: {}'.format(filename, target))
+        source_dir = source_dir or '.'
         source_url = '%s/%s' % (source_dir, filename)
+        self.logger.debug('xrootd put: source url: {}'.format(source_url))
         path = self.path2pfn(target)
         if not os.path.exists(source_url):
             raise exception.SourceNotFound()
         try:
             cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
+            self.logger.info('xrootd.put: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            if not status == 0:
+            if status != 0:
                 raise exception.RucioException(err)
         except Exception as e:
             raise exception.ServiceUnavailable(e)
@@ -233,13 +265,15 @@ def delete(self, pfn):
             :raises ServiceUnavailable: if some generic error occured in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
+        self.logger.debug('xrootd.delete: pfn: {}'.format(pfn))
         if not self.exists(pfn):
             raise exception.SourceNotFound()
         try:
             path = self.pfn2path(pfn)
             cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
+            self.logger.info('xrootd.delete: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            if not status == 0:
+            if status != 0:
                 raise exception.RucioException(err)
         except Exception as e:
             raise exception.ServiceUnavailable(e)
@@ -253,6 +287,7 @@ def rename(self, pfn, new_pfn):
             :raises ServiceUnavailable: if some generic error occured in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
+        self.logger.debug('xrootd.rename: pfn: {}'.format(pfn))
         if not self.exists(pfn):
             raise exception.SourceNotFound()
         try:
@@ -260,10 +295,12 @@ def rename(self, pfn, new_pfn):
             new_path = self.pfn2path(new_pfn)
             new_dir = new_path[:new_path.rindex('/') + 1]
             cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % (self.hostname, self.port, new_dir)
+            self.logger.info('xrootd.stat: mkdir cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % (self.hostname, self.port, path, new_path)
+            self.logger.info('xrootd.stat: rename cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            if not status == 0:
+            if status != 0:
                 raise exception.RucioException(err)
         except Exception as e:
             raise exception.ServiceUnavailable(e)


Test patch: