internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: rucio/rucio

Issue:
transfers: verify_checksum badly propagated
Motivation
----------
The verify_checksum option is nested too deep in the submission JSON to FTS. Needs to be elevated one level higher.


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py
--- a/lib/rucio/daemons/conveyor/common.py
+++ b/lib/rucio/daemons/conveyor/common.py
@@ -1,17 +1,27 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2014
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2013-2018
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2018
-# - Wen Guan, <wen.guan@cern.ch>, 2014-2016
-# - Joaquin Bogado, <jbogadog@cern.ch>, 2016
-# - Martin Barisits, <martin.barisits@cern.ch>, 2017
-# - Eric Vaandering, <ewv@fnal.gov>, 2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
+# - Vincent Garonne <vgaronne@gmail.com>, 2014-2016
+# - Martin Barisits <martin.barisits@cern.ch>, 2014-2017
+# - Wen Guan <wguan.icedew@gmail.com>, 2014-2016
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2016
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2016
+# - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
+# - Eric Vaandering <ericvaandering@gmail.com>, 2018
 
 """
 Methods common to different conveyor submitter daemons.
@@ -148,7 +158,7 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_str
             grouped_transfers[external_host] = {}
             grouped_jobs[external_host] = []
 
-        verify_checksum = transfer.get('verify_checksum', 'both')
+        verify_checksum = transfer['file_metadata'].get('verify_checksum', 'both')
         file = {'sources': transfer['sources'],
                 'destinations': transfer['dest_urls'],
                 'metadata': transfer['file_metadata'],


Test patch:
Repository: rucio/rucio

Issue:
transfers: verify_checksum badly propagated
Motivation
----------
The verify_checksum option is nested too deep in the submission JSON to FTS. Needs to be elevated one level higher.


Discussion:

Resolving patch:
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.16.2',
-    'branch_nick': 'patch-0-1_16_2_preparation',
-    'revision_id': '84967b6a13c7e10cbf2b842997dc0bccd81d4f69',
-    'revno': 6615
+    'version': '1.16.3',
+    'branch_nick': 'patch-0-Rucio_1_16_3_preparation',
+    'revision_id': '7ccd179fe23a0295f84ea530ab3612618b470001',
+    'revno': 6659
 }


Test patch:
Repository: rucio/rucio

Issue:
clients: download api, logger
Motivation
----------

STAGEIN FAILED: Get error: Staging input file failed: lfn=EVNT.04972714._000032.pool.root.1, error=stageIn with API faied: Logger instance has no attribute 'getChild'

Modification
------------

can be realted to:
https://github.com/dsoprea/GDriveFS/issues/104




Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -50,8 +50,8 @@ def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
         :param logger: logging.Logger object to use for downloads. If None nothing will be logged.
         """
         if not logger:
-            logger = logging.getLogger(__name__).getChild('null')
-            logger.addHandler(logging.NullHandler())
+            logger = logging.getLogger('%s.null' % __name__)
+            logger.disabled = True
 
         self.logger = logger
         self.is_human_readable = True
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -49,8 +49,8 @@ def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
         :param logger:      - logging.Logger object to use for uploads. If None nothing will be logged.
         """
         if not logger:
-            logger = logging.getLogger(__name__).getChild('null')
-            logger.addHandler(logging.NullHandler())
+            logger = logging.getLogger('%s.null' % __name__)
+            logger.disabled = True
 
         self.logger = logger
         self.client = _client if _client else Client()


Test patch:
Repository: rucio/rucio

Issue:
stream=True should only be used for list_replicas POST calls
Motivation
----------
At the moment it is used for ALL POST calls, but it should only be done for the list_replica one where it is needed.


Discussion:
@mlassnig @nikmagini @TomasJavurek can one of you take over this ticket? It would actually be nice if we can get this into 1.17.0 (on Monday). If nobody has time I can take it as well.

Resolving patch:
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.16.3',
-    'branch_nick': 'patch-0-Rucio_1_16_3_preparation',
-    'revision_id': '7ccd179fe23a0295f84ea530ab3612618b470001',
-    'revno': 6659
+    'version': '1.16.4',
+    'branch_nick': 'patch-0-1_16_4_client_release_preparation',
+    'revision_id': '58a9a0a76c54f0797804aecc00249fb951e7754a',
+    'revno': 6686
 }


Test patch:
Repository: rucio/rucio

Issue:
docker builds fail with pip install error (pycurl/m2crypto)
Motivation
----------
Currently the docker server and daemon builds fail while installing rucio because of the pycurl dependency needing openssl-devel installed.

Modification
------------
add yum install openssl-devel.x86_64


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/dumper/__init__.py b/lib/rucio/common/dumper/__init__.py
--- a/lib/rucio/common/dumper/__init__.py
+++ b/lib/rucio/common/dumper/__init__.py
@@ -273,7 +273,9 @@ def agis_endpoints_data(cache=True):
 def ddmendpoint_url(ddmendpoint):
     agisdata = agis_endpoints_data()
     endpointdata = next(entry for entry in agisdata if entry['name'] == ddmendpoint)
-    return str(''.join((endpointdata['se'], endpointdata['endpoint'])))
+    protocoldata = endpointdata['arprotocols']['read_wan'][0]
+    return str(''.join([protocoldata['endpoint'],
+                       re.sub(r'rucio/$', '', protocoldata['path'])]))
 
 # Using AGIS is better to avoid the use of ATLAS credentials, but using
 # rucio.client for the same task is also possible:
diff --git a/lib/rucio/daemons/auditor/srmdumps.py b/lib/rucio/daemons/auditor/srmdumps.py
--- a/lib/rucio/daemons/auditor/srmdumps.py
+++ b/lib/rucio/daemons/auditor/srmdumps.py
@@ -244,6 +244,7 @@ def download_rse_dump(rse, configuration, date='latest', destdir=DUMPS_CACHE_DIR
     logger = logging.getLogger('auditor.srmdumps')
     base_url, url_pattern = generate_url(rse, configuration)
     if date == 'latest':
+        logger.debug('Looking for site dumps in: "%s"', base_url)
         links = get_links(base_url)
         url, date = get_newest(base_url, url_pattern, links)
     else:


Test patch:
diff --git a/lib/rucio/tests/test_dumper.py b/lib/rucio/tests/test_dumper.py
--- a/lib/rucio/tests/test_dumper.py
+++ b/lib/rucio/tests/test_dumper.py
@@ -132,9 +132,15 @@ def test_agis_endpoints_data_parses_proper_json():
 
 def test_ddmendpoint_url_builds_url_from_agis_records():
     agisdata = [{
+        'arprotocols': {
+            'read_wan': [
+                {
+                    'endpoint': 'srm://example.com',
+                    'path': '/atlasdatadisk/rucio/'
+                }
+            ]
+        },
         'name': 'SOMEENDPOINT',
-        'se': 'srm://example.com/',
-        'endpoint': 'atlasdatadisk/',
     }]
     with stubbed(dumper.agis_endpoints_data, lambda: agisdata):
         eq_(dumper.ddmendpoint_url('SOMEENDPOINT'), 'srm://example.com/atlasdatadisk/')
Repository: rucio/rucio

Issue:
findall with 2 arguments failing on SL6
Motivation
----------
```
cmd: rucio get user.desilva:user.desilva.dataset.1fe078e2-a369-42c4-9dd0-3b28d95032ac
2018-06-22 07:04:23,396	INFO	Processing 1 item(s) for input
2018-06-22 07:04:23,867	ERROR	findall() takes exactly 2 arguments (3 given)
```
It seems that the definition of findall on python 2.6 and python 2.7 uses 1 and 2 arguments respectively. As we need to be compatible to python 2.6 this needs to be fixed.


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -974,13 +974,13 @@ def _parse_list_replica_metalink(self, metalink_str):
         root = ElementTree.fromstring(metalink_str)
         files = []
 
-        # metalink namespace mapping
-        ns = {'list_replicas_ml': 'urn:ietf:params:xml:ns:metalink'}
+        # metalink namespace
+        ns = '{urn:ietf:params:xml:ns:metalink}'
 
         # loop over all <file> tags of the metalink string
-        for file_ml in root.findall('list_replicas_ml:file', ns):
+        for file_ml in root.findall(ns + 'file'):
             # search for identity-tag
-            cur_did = file_ml.find('list_replicas_ml:identity', ns)
+            cur_did = file_ml.find(ns + 'identity')
             if not ElementTree.iselement(cur_did):
                 raise RucioException('Failed to locate identity-tag inside %s' % ElementTree.tostring(file_ml))
 
@@ -994,16 +994,16 @@ def _parse_list_replica_metalink(self, metalink_str):
                         'md5': None,
                         'sources': []}
 
-            size = file_ml.find('list_replicas_ml:size', ns)
+            size = file_ml.find(ns + 'size')
             if ElementTree.iselement(size):
                 cur_file['bytes'] = int(size.text)
 
-            for cur_hash in file_ml.findall('list_replicas_ml:hash', ns):
+            for cur_hash in file_ml.findall(ns + 'hash'):
                 hash_type = cur_hash.get('type')
                 if hash_type:
                     cur_file[hash_type] = cur_hash.text
 
-            for rse_ml in file_ml.findall('list_replicas_ml:url', ns):
+            for rse_ml in file_ml.findall(ns + 'url'):
                 # check if location attrib (rse name) is given
                 rse = rse_ml.get('location')
                 pfn = rse_ml.text


Test patch:
Repository: rucio/rucio

Issue:
Resurrect method doesn't remove expired_at for deleted_dids
Motivation
----------
The resurrect method doesn't remove the expired_at when a DID is resurrected which triggers the immediate deletion of the resurrected DID



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -1591,6 +1591,8 @@ def resurrect(dids, session=None):
         #    raise exception.UnsupportedOperation("File '%(scope)s:%(name)s' cannot be resurrected" % did)
 
         kargs = del_did.to_dict()
+        if kargs['expired_at']:
+            kargs['expired_at'] = None
         kargs.pop("_sa_instance_state", None)
 
         session.query(models.DeletedDataIdentifier).\


Test patch:
Repository: rucio/rucio

Issue:
Handle RSEChecksumUnavailable for protocols without renaming
Motivation
----------

For #1149 we introduced the `RSEChecksumUnavailable` to make the rucio upload client respect the `verify_checksum` RSE flag. The same exception handling should be done for protocols that do not support renaming, i.e. when `protocol.renaming == False`

Modification
------------
 - [x] `rse.rsemanager` handles `RSEChecksumUnavailable` when `protocol.renaming == False`


Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -460,13 +460,17 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
 
                 valid = None
                 try:  # Get metadata of file to verify if upload was successful
-                    stats = protocol.stat(pfn)
-                    if ('adler32' in stats) and ('adler32' in lfn):
-                        valid = stats['adler32'] == lfn['adler32']
-                    if (valid is None) and ('filesize' in stats) and ('filesize' in lfn):
-                        valid = stats['filesize'] == lfn['filesize']
-                except NotImplementedError:
-                    valid = True  # If the protocol doesn't support stat of a file, we agreed on assuming that the file was uploaded without error
+                    try:
+                        stats = protocol.stat(pfn)
+                        if ('adler32' in stats) and ('adler32' in lfn):
+                            valid = stats['adler32'] == lfn['adler32']
+                        if (valid is None) and ('filesize' in stats) and ('filesize' in lfn):
+                            valid = stats['filesize'] == lfn['filesize']
+                    except exception.RSEChecksumUnavailable as e:
+                        if rse_settings['verify_checksum'] is False:
+                            valid = True
+                        else:
+                            raise
                 except Exception as e:
                     gs = False
                     ret['%s:%s' % (scope, name)] = e


Test patch:
Repository: rucio/rucio

Issue:
core+client: switch to enable archive listing
Motivation
----------
Only resolve parent replicas if switch is True.


Discussion:

Resolving patch:
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.17.1',
-    'branch_nick': 'patch-0-1_17_1_preparation',
-    'revision_id': 'e248b33faf762729778d8465e0093aab0529a08d',
-    'revno': 6799
+    'version': '1.17.2',
+    'branch_nick': 'patch-0-1_17_2_preparation',
+    'revision_id': '9d4ae5743ac3e19100ea267021e8ebd3704672ac',
+    'revno': 6807
 }


Test patch:
Repository: rucio/rucio

Issue:
common/utils fails when loading dysfunctional paramiko version
Motivation
----------
Paramiko needs to be imported in a try/except block.



Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -97,7 +97,10 @@
     from web import HTTPError
 
 if EXTRA_MODULES['paramiko']:
-    from paramiko import RSAKey
+    try:
+        from paramiko import RSAKey
+    except Exception:
+        EXTRA_MODULES['paramiko'] = False
 
 if EXTRA_MODULES['flask']:
     from flask import Response
@@ -660,7 +663,7 @@ def ssh_sign(private_key, message):
     :return: Base64 encoded signature as a string.
     """
     if not EXTRA_MODULES['paramiko']:
-        raise MissingModuleException('The paramiko module is not installed.')
+        raise MissingModuleException('The paramiko module is not installed or faulty.')
     sio_private_key = StringIO(private_key)
     priv_k = RSAKey.from_private_key(sio_private_key)
     sio_private_key.close()


Test patch:
Repository: rucio/rucio

Issue:
common/utils fails when loading dysfunctional paramiko version
Motivation
----------
Paramiko needs to be imported in a try/except block.



Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -96,7 +96,10 @@
     from web import HTTPError
 
 if EXTRA_MODULES['paramiko']:
-    from paramiko import RSAKey
+    try:
+        from paramiko import RSAKey
+    except Exception:
+        EXTRA_MODULES['paramiko'] = False
 
 if EXTRA_MODULES['flask']:
     from flask import Response
@@ -659,7 +662,7 @@ def ssh_sign(private_key, message):
     :return: Base64 encoded signature as a string.
     """
     if not EXTRA_MODULES['paramiko']:
-        raise MissingModuleException('The paramiko module is not installed.')
+        raise MissingModuleException('The paramiko module is not installed or faulty.')
     sio_private_key = StringIO(private_key)
     priv_k = RSAKey.from_private_key(sio_private_key)
     sio_private_key.close()
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.17.2',
-    'branch_nick': 'patch-0-1_17_2_preparation',
-    'revision_id': '9d4ae5743ac3e19100ea267021e8ebd3704672ac',
-    'revno': 6807
+    'version': '1.17.3',
+    'branch_nick': 'patch-0-1_17_3_preparation',
+    'revision_id': '22a6b717ab076d8e6593d52aab6fadbe8474488a',
+    'revno': 6819
 }


Test patch:
Repository: rucio/rucio

Issue:
add --resolve-archive option to cli download
Motivation
----------

Modification
------------



Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -264,6 +264,7 @@ def download_dids(self, items, num_threads=2, trace_custom_fields={}):
         :param items: List of dictionaries. Each dictionary describing an item to download. Keys:
             did                 - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
             rse                 - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
+            resolve_archives    - Optional: bool indicating whether archives should be considered for download (Default: False)
             force_scheme        - Optional: force a specific scheme to download this item. (Default: None)
             base_dir            - Optional: base directory where the downloaded files will be stored. (Default: '.')
             no_subdir           - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
@@ -900,6 +901,7 @@ def _prepare_items_for_download(self, items):
                                                      schemes=schemes,
                                                      rse_expression=item.get('rse'),
                                                      client_location=self.client_location,
+                                                     resolve_archives=item.get('resolve_archives', False),
                                                      metalink=True)
             files_with_pfns = self._parse_list_replica_metalink(metalink_str)
 
@@ -971,7 +973,19 @@ def _split_did_str(self, did_str):
         return did_scope, did_name
 
     def _parse_list_replica_metalink(self, metalink_str):
-        root = ElementTree.fromstring(metalink_str)
+        """
+        Parses the metalink string that list_replicas can return into a list of dictionaries.
+        (This function is meant to be used as class internal only)
+
+        :param metalink_str: the metalink string to be parsed
+
+        :returns: a list with a dictionary for each file
+        """
+        try:
+            root = ElementTree.fromstring(metalink_str)
+        except Exception as error:
+            self.logger.debug(metalink_str)
+            raise error
         files = []
 
         # metalink namespace


Test patch:
Repository: rucio/rucio

Issue:
rest: metalink in replica frontend badly constructed
Motivation
----------

The metalink frontend is still using the old RSE scheme instead of the new PFN scheme.


Discussion:

Resolving patch:
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -357,10 +357,9 @@ def POST(self):
 
                     replicas = []
                     dictreplica = {}
-                    for rse in rfile['rses']:
-                        for replica in rfile['rses'][rse]:
-                            replicas.append(replica)
-                            dictreplica[replica] = (rfile['pfns'][replica]['domain'], rfile['pfns'][replica]['priority'], rse)
+                    for replica in rfile['pfns'].keys():
+                        replicas.append(replica)
+                        dictreplica[replica] = (rfile['pfns'][replica]['domain'], rfile['pfns'][replica]['priority'], rfile['pfns'][replica]['rse'])
 
                     yield ' <file name="' + rfile['name'] + '">\n'
                     yield '  <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'


Test patch: