internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: rucio/rucio

Issue:
add --resolve-archive option to cli download
Motivation
----------

Modification
------------



Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -264,6 +264,7 @@ def download_dids(self, items, num_threads=2, trace_custom_fields={}):
         :param items: List of dictionaries. Each dictionary describing an item to download. Keys:
             did                 - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
             rse                 - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
+            resolve_archives    - Optional: bool indicating whether archives should be considered for download (Default: False)
             force_scheme        - Optional: force a specific scheme to download this item. (Default: None)
             base_dir            - Optional: base directory where the downloaded files will be stored. (Default: '.')
             no_subdir           - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
@@ -900,6 +901,7 @@ def _prepare_items_for_download(self, items):
                                                      schemes=schemes,
                                                      rse_expression=item.get('rse'),
                                                      client_location=self.client_location,
+                                                     resolve_archives=item.get('resolve_archives', False),
                                                      metalink=True)
             files_with_pfns = self._parse_list_replica_metalink(metalink_str)
 
@@ -971,7 +973,19 @@ def _split_did_str(self, did_str):
         return did_scope, did_name
 
     def _parse_list_replica_metalink(self, metalink_str):
-        root = ElementTree.fromstring(metalink_str)
+        """
+        Parses the metalink string that list_replicas can return into a list of dictionaries.
+        (This function is meant to be used as class internal only)
+
+        :param metalink_str: the metalink string to be parsed
+
+        :returns: a list with a dictionary for each file
+        """
+        try:
+            root = ElementTree.fromstring(metalink_str)
+        except Exception as error:
+            self.logger.debug(metalink_str)
+            raise error
         files = []
 
         # metalink namespace
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -372,7 +372,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
 
     protocol = create_protocol(rse_settings, 'write', scheme=force_scheme)
     protocol.connect()
-    protocol_delete = create_protocol(rse_settings, 'delete', scheme=force_scheme)
+    protocol_delete = create_protocol(rse_settings, 'delete')
     protocol_delete.connect()
 
     lfns = [lfns] if not type(lfns) is list else lfns


Test patch:
Repository: rucio/rucio

Issue:
add --resolve-archive option to cli download
Motivation
----------

Modification
------------



Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -264,6 +264,7 @@ def download_dids(self, items, num_threads=2, trace_custom_fields={}):
         :param items: List of dictionaries. Each dictionary describing an item to download. Keys:
             did                 - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
             rse                 - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
+            resolve_archives    - Optional: bool indicating whether archives should be considered for download (Default: False)
             force_scheme        - Optional: force a specific scheme to download this item. (Default: None)
             base_dir            - Optional: base directory where the downloaded files will be stored. (Default: '.')
             no_subdir           - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
@@ -900,6 +901,7 @@ def _prepare_items_for_download(self, items):
                                                      schemes=schemes,
                                                      rse_expression=item.get('rse'),
                                                      client_location=self.client_location,
+                                                     resolve_archives=item.get('resolve_archives', False),
                                                      metalink=True)
             files_with_pfns = self._parse_list_replica_metalink(metalink_str)
 
@@ -971,7 +973,19 @@ def _split_did_str(self, did_str):
         return did_scope, did_name
 
     def _parse_list_replica_metalink(self, metalink_str):
-        root = ElementTree.fromstring(metalink_str)
+        """
+        Parses the metalink string that list_replicas can return into a list of dictionaries.
+        (This function is meant to be used as class internal only)
+
+        :param metalink_str: the metalink string to be parsed
+
+        :returns: a list with a dictionary for each file
+        """
+        try:
+            root = ElementTree.fromstring(metalink_str)
+        except Exception as error:
+            self.logger.debug(metalink_str)
+            raise error
         files = []
 
         # metalink namespace
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.17.3',
-    'branch_nick': 'patch-0-1_17_3_preparation',
-    'revision_id': '22a6b717ab076d8e6593d52aab6fadbe8474488a',
-    'revno': 6819
+    'version': '1.17.4',
+    'branch_nick': 'patch-0-1_17_4_client_release_prep',
+    'revision_id': 'ba996ce9bf8366cd7d8d1fb60a7f1daf8d4f517e',
+    'revno': 6827
 }


Test patch:
Repository: rucio/rucio

Issue:
Downgrade to older setuptools version as 39.0.1 is not py2.6 compatible
Motivation
----------
Need to downgrade to 36.8.0 (Last py2.6 supported version)


Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -372,7 +372,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
 
     protocol = create_protocol(rse_settings, 'write', scheme=force_scheme)
     protocol.connect()
-    protocol_delete = create_protocol(rse_settings, 'delete', scheme=force_scheme)
+    protocol_delete = create_protocol(rse_settings, 'delete')
     protocol_delete.connect()
 
     lfns = [lfns] if not type(lfns) is list else lfns


Test patch:
Repository: rucio/rucio

Issue:
Fix activity in BB8
Motivation
----------

BB8 uses activity `Data Rebalancing` but the activity defined in ATLAS schema is `Data rebalancing`. We should use the same activity everywhere, and it should be consistent with the share defined in FTS


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/bb8/common.py b/lib/rucio/daemons/bb8/common.py
--- a/lib/rucio/daemons/bb8/common.py
+++ b/lib/rucio/daemons/bb8/common.py
@@ -243,7 +243,7 @@ def rebalance_rse(rse, max_bytes=1E9, max_files=None, dry_run=False, exclude_exp
                 # Rebalance this rule
                 if not dry_run:
                     child_rule_id = rebalance_rule(parent_rule_id=rule_id,
-                                                   activity='Data Rebalancing',
+                                                   activity='Data rebalancing',
                                                    rse_expression=target_rse_exp,
                                                    priority=priority,
                                                    source_replica_expression=source_replica_expression,


Test patch:
Repository: rucio/rucio

Issue:
Upload fails to delete left-overs
Motivation
----------
deletion left-overs of previous attempts of upload is silently failing.
https://github.com/rucio/rucio/blob/master/lib/rucio/rse/rsemanager.py#L409
consequently, upload of current attempt fails. 

Modification
------------
Under investigation. 


Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -406,9 +406,11 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
             else:
                 if protocol.exists('%s.rucio.upload' % pfn):  # Check for left over of previous unsuccessful attempts
                     try:
-                        protocol_delete.delete('%s.rucio.upload', list(protocol_delete.lfns2pfns(make_valid_did(lfn)).values())[0])
+                        protocol_delete.delete('%s.rucio.upload' % list(protocol_delete.lfns2pfns(make_valid_did(lfn)).values())[0])
                     except Exception as e:
                         ret['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('Unable to remove temporary file %s.rucio.upload: %s' % (pfn, str(e)))
+                        gs = False
+                        continue
                 try:  # Try uploading file
                     protocol.put(base_name, '%s.rucio.upload' % pfn, source_dir, transfer_timeout=transfer_timeout)
                 except Exception as e:


Test patch:
Repository: rucio/rucio

Issue:
Fix activity in BB8
Motivation
----------

BB8 uses activity `Data Rebalancing` but the activity defined in ATLAS schema is `Data rebalancing`. We should use the same activity everywhere, and it should be consistent with the share defined in FTS


Discussion:
This doesn't seem to be my fault:
with python 2.7:

CannotAuthenticate: Cannot authenticate.

Although it can be, that certain activities can be processed only by certain accounts, and travis had not the rights to run this activity?


Resolving patch:
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.17.4',
-    'branch_nick': 'patch-0-1_17_4_client_release_prep',
-    'revision_id': 'ba996ce9bf8366cd7d8d1fb60a7f1daf8d4f517e',
-    'revno': 6827
+    'version': '1.17.5',
+    'branch_nick': 'patch-0-1_17_5_preparation',
+    'revision_id': '537e1e47eb627741394b6bb9bc21d0f046296275',
+    'revno': 6837
 }


Test patch:
Repository: rucio/rucio

Issue:
Replacing single 'raise' from rsemgr by 'raise e'
https://github.com/rucio/rucio/blob/b716392d1dee3e0ee8ec87a87df4c2e02b7781db/lib/rucio/rse/rsemanager.py#L473

change:
raise -> raise exception


Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -428,7 +428,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                         if rse_settings['verify_checksum'] is False:
                             valid = True
                         else:
-                            raise
+                            raise exception.RucioException('Checksum not validated')
                 except Exception as e:
                     gs = False
                     ret['%s:%s' % (scope, name)] = e
@@ -470,7 +470,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                         if rse_settings['verify_checksum'] is False:
                             valid = True
                         else:
-                            raise
+                            raise exception.RucioException('Checksum not validated')
                 except Exception as e:
                     gs = False
                     ret['%s:%s' % (scope, name)] = e


Test patch:
Repository: rucio/rucio

Issue:
Replacing single 'raise' from rsemgr by 'raise e'
https://github.com/rucio/rucio/blob/b716392d1dee3e0ee8ec87a87df4c2e02b7781db/lib/rucio/rse/rsemanager.py#L473

change:
raise -> raise exception


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -531,7 +531,7 @@ def download(self, dids, rse=None, archive=None, protocol=None, pfn=None, nrando
         OBSOLETE! This function is kept for compability reasons and will be removed in a future release!
         """
         item_tpl = {'rse': rse,
-                    'force_scheme': [protocol],
+                    'force_scheme': protocol,
                     'nrandom': nrandom,
                     'base_dir': dir,
                     'no_subdir': no_subd,


Test patch:
Repository: rucio/rucio

Issue:
Extend the bad_replicas table with bytes
Motivation
----------

The bad_replicas table doesn't contain the bytes associated to lost or corrupted files, making it more complicated to provide summaries on lost files.


Discussion:

Resolving patch:
diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py b/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py
@@ -0,0 +1,46 @@
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2018
+# Add bytes column to bad_replicas
+#
+# Revision ID: 1f46c5f240ac
+# Revises: 688ef1840840
+# Create Date: 2018-07-26 15:28:04.243520
+
+from alembic.op import add_column, drop_column
+from alembic import context
+
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '1f46c5f240ac'  # pylint: disable=invalid-name
+down_revision = '688ef1840840'  # pylint: disable=invalid-name
+
+
+def upgrade():
+    '''
+    upgrade method
+    '''
+    if context.get_context().dialect.name != 'sqlite':  # pylint: disable=no-member
+        add_column('bad_replicas', sa.Column('bytes', sa.BigInteger))
+
+
+def downgrade():
+    '''
+    downgrade method
+    '''
+    if context.get_context().dialect.name != 'sqlite':  # pylint: disable=no-member
+        drop_column('bad_replicas', 'bytes')
diff --git a/lib/rucio/db/sqla/models.py b/lib/rucio/db/sqla/models.py
--- a/lib/rucio/db/sqla/models.py
+++ b/lib/rucio/db/sqla/models.py
@@ -454,6 +454,7 @@ class BadReplicas(BASE, ModelBase):
     reason = Column(String(255))
     state = Column(BadFilesStatus.db_type(name='BAD_REPLICAS_STATE_CHK'), default=BadFilesStatus.SUSPICIOUS)
     account = Column(String(25))
+    bytes = Column(BigInteger)
     _table_args = (PrimaryKeyConstraint('scope', 'name', 'rse_id', 'created_at', name='BAD_REPLICAS_STATE_PK'),
                    CheckConstraint('SCOPE IS NOT NULL', name='BAD_REPLICAS_SCOPE_NN'),
                    CheckConstraint('NAME IS NOT NULL', name='BAD_REPLICAS_NAME_NN'),


Test patch:
Repository: rucio/rucio

Issue:
readthedocs build fails due to pycurl
Motivation
----------
Currently the build process on readthedocs fails.

```
Collecting pycurl>=7.19 (from fts3-rest-API==3.7.1->-r requirements.readthedocs.txt (line 44))
  Using cached https://files.pythonhosted.org/packages/e8/e4/0dbb8735407189f00b33d84122b9be52c790c7c3b25286826f4e1bdb7bde/pycurl-7.43.0.2.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/tmp/pip-build-jSh8ZH/pycurl/setup.py", line 913, in <module>
        ext = get_extension(sys.argv, split_extension_source=split_extension_source)
      File "/tmp/pip-build-jSh8ZH/pycurl/setup.py", line 582, in get_extension
        ext_config = ExtensionConfiguration(argv)
      File "/tmp/pip-build-jSh8ZH/pycurl/setup.py", line 99, in __init__
        self.configure()
      File "/tmp/pip-build-jSh8ZH/pycurl/setup.py", line 227, in configure_unix
        raise ConfigurationError(msg)
    __main__.ConfigurationError: Could not run curl-config: [Errno 2] No such file or directory
    
    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in /tmp/pip-build-jSh8ZH/pycurl/
You are using pip version 9.0.1, however version 18.0 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
```

Lifetime exception request interface doesn't properly show errors
Motivation
----------
Lifetime exception request interface doesn't properly show errors



Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -406,9 +406,11 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
             else:
                 if protocol.exists('%s.rucio.upload' % pfn):  # Check for left over of previous unsuccessful attempts
                     try:
-                        protocol_delete.delete('%s.rucio.upload', list(protocol_delete.lfns2pfns(make_valid_did(lfn)).values())[0])
+                        protocol_delete.delete('%s.rucio.upload' % list(protocol_delete.lfns2pfns(make_valid_did(lfn)).values())[0])
                     except Exception as e:
                         ret['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('Unable to remove temporary file %s.rucio.upload: %s' % (pfn, str(e)))
+                        gs = False
+                        continue
                 try:  # Try uploading file
                     protocol.put(base_name, '%s.rucio.upload' % pfn, source_dir, transfer_timeout=transfer_timeout)
                 except Exception as e:


Test patch:
Repository: rucio/rucio

Issue:
Bug due to move to heartbeat
Motivation
----------
The heart_beat in the conveyor is done using the full executable + arguments. One should only keep the executable + the activity option.
In addition a bug has been introduced by moving from `type(request_type) is not list`  to `isinstance(request_type, list)`



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -210,7 +210,7 @@ def get_next_transfers_new(request_type, state, limit=100, older_than=None, rse=
     record_counter('core.request.get_next_transfers.%s-%s' % (request_type, state))
 
     # lists of one element are not allowed by SQLA, so just duplicate the item
-    if isinstance(request_type, list):
+    if not isinstance(request_type, list):
         request_type = [request_type, request_type]
     elif len(request_type) == 1:
         request_type = [request_type[0], request_type[0]]
diff --git a/lib/rucio/daemons/conveyor/poller.py b/lib/rucio/daemons/conveyor/poller.py
--- a/lib/rucio/daemons/conveyor/poller.py
+++ b/lib/rucio/daemons/conveyor/poller.py
@@ -72,7 +72,14 @@ def poller(once=False, activities=None, sleep_time=60,
     except NoOptionError:
         timeout = None
 
-    executable = ' '.join(sys.argv)
+    executable = sys.argv[0]
+    if activities:
+        activities.sort()
+        executable += '--activities ' + str(activities)
+    if activity_shares:
+        activities.sort()
+        executable += '--activity_shares' + str(activity_shares)
+
     hostname = socket.getfqdn()
     pid = os.getpid()
     hb_thread = threading.current_thread()
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -97,7 +97,11 @@ def submitter(once=False, rses=None, mock=False,
     logging.debug("Maximum time in queue for different activities: %s" % max_time_in_queue)
 
     activity_next_exe_time = defaultdict(time.time)
-    executable = ' '.join(sys.argv)
+    executable = sys.argv[0]
+    if activities:
+        activities.sort()
+        executable += '--activities ' + str(activities)
+
     hostname = socket.getfqdn()
     pid = os.getpid()
     hb_thread = threading.current_thread()


Test patch:
Repository: rucio/rucio

Issue:
Add a --allow-tape option in rucio download
Motivation
----------
`rucio download` does a check at the beginning to see if the account is an admin one in case the replica is on tape. It puts unneeded load on the servers.

Modification
------------
Propose to forbid any access to replicas on TAPE by defaut and add a `--alllowtape` option. If this option is set to True, then the admin check is done



Discussion:
suggestion: "--allow-tape" :-)


Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -45,7 +45,7 @@
 
 class DownloadClient:
 
-    def __init__(self, client=None, logger=None):
+    def __init__(self, client=None, logger=None, check_admin=False):
         """
         Initialises the basic settings for an DownloadClient object
 
@@ -62,12 +62,13 @@ def __init__(self, client=None, logger=None):
 
         self.client_location = detect_client_location()
 
-        account_attributes = [acc for acc in self.client.list_account_attributes(self.client.account)]
         self.is_admin = False
-        for attr in account_attributes[0]:
-            if attr['key'] == 'admin':
-                self.is_admin = attr['value'] is True
-                break
+        if check_admin:
+            account_attributes = list(self.client.list_account_attributes(self.client.account))
+            for attr in account_attributes[0]:
+                if attr['key'] == 'admin':
+                    self.is_admin = attr['value'] is True
+                    break
         if self.is_admin:
             logger.debug('Admin mode enabled')
 


Test patch:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -36,6 +36,7 @@
 from rucio.client.replicaclient import ReplicaClient
 from rucio.common.config import config_get
 from rucio.common.utils import generate_uuid, md5
+from rucio.core.rse import add_rse_attribute
 from rucio.tests.common import execute, account_name_generator, rse_name_generator, file_generator, scope_name_generator
 from rucio.rse import rsemanager as rsemgr
 
@@ -58,6 +59,8 @@ def setup(self):
         self.account_client = AccountLimitClient()
         self.account_client.set_account_limit('root', self.def_rse, -1)
 
+        add_rse_attribute(self.def_rse, 'istape', 'False')
+
         self.upload_success_str = 'Successfully uploaded file %s'
 
     def test_rucio_version(self):