Repository: rucio/rucio
Issue: dids: setting events with set_metadata fails

Motivation
----------
(cx_Oracle.NotSupportedError) Variable_TypeByValue(): unhandled data type result [SQL: u'UPDATE atlas_rucio.dids SET events=:events, updated_at=:updated_at WHERE atlas_rucio.dids.scope = :scope_1 AND atlas_rucio.dids.name = :name_1'] [parameters: {'events': (1000,), u'name_1': 'mc16_13TeV.364186.Sherpa_221_NNPDF30NNLO_Wtaunu_MAXHTPTV0_70_BFilter.simul.HITS.e5340_e5984_s3126_tid13424328_00', 'updated_at': datetime.datetime(2018, 3, 16, 11, 13, 6, 247121), u'scope_1': 'mc16_13TeV'}] (Background on this error at: http://sqlalche.me/e/tw8g)

Modification
------------

...one() --> .one()[0]

Changed files:
lib/rucio/rse/protocols/gfal.py, lib/rucio/core/did.py, lib/rucio/rse/protocols/posix.py, lib/rucio/rse/rsemanager.py, setup_rucio_client.py, lib/rucio/rse/protocols/gsiftp.py, lib/rucio/rse/protocols/ngarc.py, lib/rucio/rse/protocols/srm.py, lib/rucio/rse/protocols/sftp.py

diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
index b8caf00cb..3e6ee7959 100644
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -1,22 +1,27 @@
-'''
-  Copyright European Organization for Nuclear Research (CERN)
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  You may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Authors:
-  - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2018
-  - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2015, 2017
-  - Yun-Pin Sun, <yun-pin.sun@cern.ch>, 2013
-  - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2015, 2017
-  - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017
-  - Ralph Vigne, <ralph.vigne@cern.ch>, 2013
-  - Thomas Beermann, <thomas.beermann@cern.ch>, 2014, 2016-2018
-  - Joaquin Bogado, <joaquin.bogado@cern.ch>, 2014-2015
-  - Wen Guan, <wen.guan@cern.ch>, 2015
-'''
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
+# - Ralph Vigne <ralph.vigne@cern.ch>, 2013
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
+# - Yun-Pin Sun <winter0128@gmail.com>, 2013
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2015
+# - Wen Guan <wguan.icedew@gmail.com>, 2015
 
 import logging
 import random
@@ -1170,7 +1175,7 @@ def set_metadata(scope, name, key, value, type=None, did=None,
         session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
         if key == 'events':
             for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
-                events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()
+                events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()[0]
                 session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update({'events': events}, synchronize_session=False)
 
     elif key == 'adler32':
diff --git a/lib/rucio/rse/protocols/gfal.py b/lib/rucio/rse/protocols/gfal.py
index 062887eb1..242dbf727 100644
--- a/lib/rucio/rse/protocols/gfal.py
+++ b/lib/rucio/rse/protocols/gfal.py
@@ -13,6 +13,7 @@
  - Mario Lassnig, <mario.lassnig@cern.ch>, 2016-2017
  - Tobias Wegner, <tobias.wegner@cern.ch>, 2017
  - Nicolo Magini, <nicolo.magini@cern.ch>, 2018
+ - Joaquin Bogado, <jbogado@linti.unlp.edu.ar>, 2018
 '''
 
 import errno
@@ -45,7 +46,7 @@ class Default(protocol.RSEProtocol):
 
         pfns = {}
         prefix = self.attributes['prefix']
-        if self.attributes['extended_attributes'] is not None and 'web_service_path' in self.attributes['extended_attributes'].keys():
+        if self.attributes['extended_attributes'] is not None and 'web_service_path' in list(self.attributes['extended_attributes'].keys()):
             web_service_path = self.attributes['extended_attributes']['web_service_path']
         else:
             web_service_path = ''
@@ -142,7 +143,7 @@ class Default(protocol.RSEProtocol):
         if '://' in hostname:
             hostname = hostname.split("://")[1]
 
-        if 'extended_attributes' in self.attributes.keys() and self.attributes['extended_attributes'] is not None and 'web_service_path' in self.attributes['extended_attributes'].keys():
+        if 'extended_attributes' in list(self.attributes.keys()) and self.attributes['extended_attributes'] is not None and 'web_service_path' in list(self.attributes['extended_attributes'].keys()):
             web_service_path = self.attributes['extended_attributes']['web_service_path']
         else:
             web_service_path = ''
@@ -218,7 +219,7 @@ class Default(protocol.RSEProtocol):
             source_url = "file://" + source_url
 
         space_token = None
-        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
+        if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
             space_token = self.attributes['extended_attributes']['space_token']
 
         try:
@@ -462,7 +463,7 @@ class Default(protocol.RSEProtocol):
         """
         endpoint_basepath = self.path2pfn(self.attributes['prefix'])
         space_token = None
-        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
+        if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
             space_token = self.attributes['extended_attributes']['space_token']
 
         if space_token is None or space_token == "":
diff --git a/lib/rucio/rse/protocols/gsiftp.py b/lib/rucio/rse/protocols/gsiftp.py
index 0e0bffbd3..e3992fea3 100644
--- a/lib/rucio/rse/protocols/gsiftp.py
+++ b/lib/rucio/rse/protocols/gsiftp.py
@@ -84,7 +84,7 @@ class Default(protocol.RSEProtocol):
                 data_file = open(dest)
                 data = json.load(data_file)
                 data_file.close()
-                if agis_token not in data.keys():
+                if agis_token not in list(data.keys()):
                     print('ERROR: space usage json has different token as key')
                 else:
                     totalsize = int(data[agis_token]['total_space'])
diff --git a/lib/rucio/rse/protocols/ngarc.py b/lib/rucio/rse/protocols/ngarc.py
index 3dcfa0ea2..e34e0d372 100644
--- a/lib/rucio/rse/protocols/ngarc.py
+++ b/lib/rucio/rse/protocols/ngarc.py
@@ -1,21 +1,28 @@
-"""
- Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - David Cameron <david.cameron@cern.ch>, 2014
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2017
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 
- Licensed under the Apache License, Version 2.0 (the "License");
- You may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-
- Authors:
- - David Cameron <david.cameron@cern.ch>, 2014
- - Cedric Serfon <cedric.serfon@cern.ch>, 2017
-"""
 
 import errno
 import os
 
 try:
-    import arc
+    import arc  # pylint: disable=import-error
 except:
     pass
 
@@ -161,7 +168,7 @@ class Default(protocol.RSEProtocol):
             sf = source
 
         space_token = None
-        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
+        if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
             space_token = self.attributes['extended_attributes']['space_token']
 
         self.__arc_copy(sf, target, space_token)
diff --git a/lib/rucio/rse/protocols/posix.py b/lib/rucio/rse/protocols/posix.py
index ca7fbd05d..3d6cc89f8 100644
--- a/lib/rucio/rse/protocols/posix.py
+++ b/lib/rucio/rse/protocols/posix.py
@@ -155,7 +155,7 @@ class Default(protocol.RSEProtocol):
                 raise exception.ServiceUnavailable(e)
 
     def pfn2path(self, pfn):
-        tmp = self.parse_pfns(pfn).values()[0]
+        tmp = list(self.parse_pfns(pfn).values())[0]
         return '/'.join([tmp['prefix'], tmp['path'], tmp['name']])
 
     def stat(self, pfn):
diff --git a/lib/rucio/rse/protocols/sftp.py b/lib/rucio/rse/protocols/sftp.py
index 2da5ced3e..bf4014d02 100644
--- a/lib/rucio/rse/protocols/sftp.py
+++ b/lib/rucio/rse/protocols/sftp.py
@@ -8,7 +8,7 @@
 # Authors:
 # - Ralph Vigne, <ralph.vigne@cern.ch>, 2012-214
 # - Yun-Pin Sun, <yun-pin.sun@cern.ch>, 2012
-
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 
 import pysftp
 from subprocess import call
@@ -162,7 +162,7 @@ class Default(protocol.RSEProtocol):
                 raise exception.ServiceUnavailable(e)
 
     def pfn2path(self, pfn):
-        tmp = self.parse_pfns(pfn).values()[0]
+        tmp = list(self.parse_pfns(pfn).values())[0]
         return '/'.join([tmp['prefix'], tmp['path'], tmp['name']])
 
     def stat(self, pfn):
diff --git a/lib/rucio/rse/protocols/srm.py b/lib/rucio/rse/protocols/srm.py
index dd1fbaf12..a2f4f9c5a 100644
--- a/lib/rucio/rse/protocols/srm.py
+++ b/lib/rucio/rse/protocols/srm.py
@@ -35,7 +35,7 @@ class Default(protocol.RSEProtocol):
         pfns = {}
         prefix = self.attributes['prefix']
         if self.attributes['extended_attributes'] is not None and\\
-           'web_service_path' in self.attributes['extended_attributes'].keys():
+           'web_service_path' in list(self.attributes['extended_attributes'].keys()):
             web_service_path = self.attributes['extended_attributes']['web_service_path']
         else:
             web_service_path = ''
@@ -148,9 +148,9 @@ class Default(protocol.RSEProtocol):
         if '://' in hostname:
             hostname = hostname.split("://")[1]
 
-        if 'extended_attributes' in self.attributes.keys() and\\
+        if 'extended_attributes' in list(self.attributes.keys()) and\\
            self.attributes['extended_attributes'] is not None and\\
-           'web_service_path' in self.attributes['extended_attributes'].keys():
+           'web_service_path' in list(self.attributes['extended_attributes'].keys()):
             web_service_path = self.attributes['extended_attributes']['web_service_path']
         else:
             web_service_path = ''
@@ -224,7 +224,7 @@ class Default(protocol.RSEProtocol):
             raise exception.SourceNotFound()
 
         space_token = ''
-        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
+        if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
             space_token = '--dst %s' % self.attributes['extended_attributes']['space_token']
 
         try:
@@ -274,7 +274,7 @@ class Default(protocol.RSEProtocol):
         """
 
         space_token = ''
-        if self.attributes['extended_attributes'] is not None and 'space_token' in self.attributes['extended_attributes'].keys():
+        if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
             space_token = '--dst %s' % self.attributes['extended_attributes']['space_token']
 
         try:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
index 424b3441c..a676d0325 100755
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -103,7 +103,7 @@ def _get_possible_protocols(rse_settings, operation, scheme=None, domain=None):
         filtered = True
 
         if not domain:
-            for d in protocol['domains'].keys():
+            for d in list(protocol['domains'].keys()):
                 if protocol['domains'][d][operation] != 0:
                     filtered = False
         else:
@@ -245,7 +245,7 @@ def download(rse_settings, files, dest_dir=None, force_scheme=None, ignore_check
 
     files = [files] if not type(files) is list else files
     for f in files:
-        pfn = f['pfn'] if 'pfn' in f else protocol.lfns2pfns(f).values()[0]
+        pfn = f['pfn'] if 'pfn' in f else list(protocol.lfns2pfns(f).values())[0]
         target_dir = "./%s" % f['scope'] if dest_dir is None else dest_dir
         try:
             if not os.path.exists(target_dir):
@@ -319,10 +319,10 @@ def exists(rse_settings, files):
             exists = protocol.exists(f)
             ret[f] = exists
         elif 'scope' in f:  # a LFN is provided
-            pfn = protocol.lfns2pfns(f).values()[0]
+            pfn = list(protocol.lfns2pfns(f).values())[0]
             if isinstance(pfn, exception.RucioException):
                 raise pfn
-            exists = protocol.exists(protocol.lfns2pfns(f).values()[0])
+            exists = protocol.exists(list(protocol.lfns2pfns(f).values())[0])
             ret[f['scope'] + ':' + f['name']] = exists
         else:
             exists = protocol.exists(f['name'])
@@ -385,7 +385,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
         if force_pfn:
             pfn = force_pfn
         else:
-            pfn = protocol.lfns2pfns(make_valid_did(lfn)).values()[0]
+            pfn = list(protocol.lfns2pfns(make_valid_did(lfn)).values())[0]
             if isinstance(pfn, exception.RucioException):
                 raise pfn
 
@@ -399,7 +399,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
             else:
                 if protocol.exists('%s.rucio.upload' % pfn):  # Check for left over of previous unsuccessful attempts
                     try:
-                        protocol_delete.delete('%s.rucio.upload', protocol_delete.lfns2pfns(make_valid_did(lfn)).values()[0])
+                        protocol_delete.delete('%s.rucio.upload', list(protocol_delete.lfns2pfns(make_valid_did(lfn)).values())[0])
                     except Exception as e:
                         ret['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('Unable to remove temporary file %s.rucio.upload: %s' % (pfn, str(e)))
                 try:  # Try uploading file
@@ -499,7 +499,7 @@ def delete(rse_settings, lfns):
 
     lfns = [lfns] if not type(lfns) is list else lfns
     for lfn in lfns:
-        pfn = protocol.lfns2pfns(lfn).values()[0]
+        pfn = list(protocol.lfns2pfns(lfn).values())[0]
         try:
             protocol.delete(pfn)
             ret['%s:%s' % (lfn['scope'], lfn['name'])] = True
@@ -557,8 +557,8 @@ def rename(rse_settings, files):
             # Check if new scope is provided
             if 'new_scope' not in f:
                 f['new_scope'] = f['scope']
-            pfn = protocol.lfns2pfns({'name': f['name'], 'scope': f['scope']}).values()[0]
-            new_pfn = protocol.lfns2pfns({'name': f['new_name'], 'scope': f['new_scope']}).values()[0]
+            pfn = list(protocol.lfns2pfns({'name': f['name'], 'scope': f['scope']}).values())[0]
+            new_pfn = list(protocol.lfns2pfns({'name': f['new_name'], 'scope': f['new_scope']}).values())[0]
         else:
             pfn = f['name']
             new_pfn = f['new_name']
diff --git a/setup_rucio_client.py b/setup_rucio_client.py
index ac492792c..1dc191680 100644
--- a/setup_rucio_client.py
+++ b/setup_rucio_client.py
@@ -30,7 +30,7 @@ sys.path.insert(0, os.path.abspath('lib/'))
 from rucio import version  # noqa
 
 if sys.version_info < (2, 5):
-    print 'ERROR: Rucio requires at least Python 2.6 to run.'
+    print('ERROR: Rucio requires at least Python 2.6 to run.')
     sys.exit(1)
 sys.path.insert(0, os.path.abspath('lib/'))
 
@@ -87,7 +87,7 @@ try:
                 self.finalize_options()
                 BuildDoc.run(self)
     cmdclass['build_sphinx'] = local_BuildDoc
-except:
+except Exception:
     pass
 
 
@@ -166,7 +166,7 @@ class CustomSdist(_sdist):
         '''
         get_file_list
         '''
-        print "Chosen packaging option: " + NAME
+        print("Chosen packaging option: " + NAME)
         self.distribution.data_files = DATA_FILES
         _sdist.get_file_list(self)