internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: rucio/rucio

Issue:
Typos and other small fixes
Motivation
----------

Stumbled upon them while using Rucio. This issue collects them, though they aren’t related.

Modification
------------

Small patches of little consequence.
Kronos crashes when updating eol_at
Motivation
----------
Kronos crashes when updating eol_at


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/rseclient.py b/lib/rucio/client/rseclient.py
--- a/lib/rucio/client/rseclient.py
+++ b/lib/rucio/client/rseclient.py
@@ -196,7 +196,7 @@ def list_rse_attributes(self, rse):
 
         :param rse: The RSE name.
 
-        :return: True if RSE attribute was created successfully else False.
+        :return: A ``dict`` with the RSE attribute name/value pairs.
         """
         path = '/'.join([self.RSE_BASEURL, rse, 'attr/'])
         url = build_url(choice(self.list_hosts), path=path)
diff --git a/lib/rucio/common/dumper/__init__.py b/lib/rucio/common/dumper/__init__.py
--- a/lib/rucio/common/dumper/__init__.py
+++ b/lib/rucio/common/dumper/__init__.py
@@ -247,7 +247,7 @@ def to_datetime(str_or_datetime):
 def agis_endpoints_data(cache=True):
     '''
     Returns the result of querying the ddmendpoints list from AGIS
-    and parsing the results with json.lads().
+    and parsing the results with json.loads().
 
     :param cache: If True only makes one query to AGIS and returns
     the same result for subsequent invocations. Consider the effects
diff --git a/lib/rucio/daemons/auditor/__init__.py b/lib/rucio/daemons/auditor/__init__.py
--- a/lib/rucio/daemons/auditor/__init__.py
+++ b/lib/rucio/daemons/auditor/__init__.py
@@ -69,7 +69,7 @@ def consistency(rse, delta, configuration, cache_dir, results_dir):
 
 
 def guess_replica_info(path):
-    """Try to extract the scope and name from a path
+    """Try to extract the scope and name from a path.
 
     ``path`` should be an ``str`` with the relative path to the file on
     the RSE.
@@ -87,7 +87,7 @@ def guess_replica_info(path):
 
 
 def bz2_compress_file(source, chunk_size=65000):
-    """Compress a file with bzip2
+    """Compress a file with bzip2.
 
     The destination is the path passed through ``source`` extended with
     '.bz2'.  The original file is deleted.
@@ -115,7 +115,7 @@ def bz2_compress_file(source, chunk_size=65000):
 
 
 def process_output(output, sanity_check=True, compress=True):
-    """Perform post-consistency-check actions
+    """Perform post-consistency-check actions.
 
     DARK files are put in the quarantined-replica table so that they
     may be deleted by the Dark Reaper.  LOST files are currently


Test patch:
Repository: rucio/rucio

Issue:
Suspicious file reporting broken in Kronos
Motivation
----------
The reporting of suspicious files in Kronos is broken



Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/tracer/kronos.py b/lib/rucio/daemons/tracer/kronos.py
--- a/lib/rucio/daemons/tracer/kronos.py
+++ b/lib/rucio/daemons/tracer/kronos.py
@@ -143,11 +143,15 @@ def __update_atime(self):
                     for pattern in self.__bad_files_patterns:
                         if 'stateReason' in report and pattern.match(report['stateReason']):
                             reason = report['stateReason'][:255]
-                            try:
-                                declare_bad_file_replicas(report['url'], reason=reason, issuer='root', status=BadFilesStatus.SUSPICIOUS)
-                                logging.info('Declare suspicious file %s with reason %s' % (report['url'], reason))
-                            except Exception as error:
-                                logging.error('Failed to declare suspicious file' + str(error))
+                            if 'url' not in report or not report['url']:
+                                logging.error('Missing url in the following trace : ' + str(report))
+                            else:
+                                try:
+                                    surl = report['url']
+                                    declare_bad_file_replicas([surl, ], reason=reason, issuer='root', status=BadFilesStatus.SUSPICIOUS)
+                                    logging.info('Declare suspicious file %s with reason %s' % (report['url'], reason))
+                                except Exception as error:
+                                    logging.error('Failed to declare suspicious file' + str(error))
 
                 # check if scope in report. if not skip this one.
                 if 'scope' not in report:


Test patch:
Repository: rucio/rucio

Issue:
Conveyor submitter fts error message wrong
Motivation
----------
In https://github.com/rucio/rucio/blob/b8a77623801da54c88b249fbc643657a0a04453d/lib/rucio/core/transfer.py#L576 the error message says the fts host is missing for the source, while it should say it is missing for the destination.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -573,7 +573,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                 # get external_host
                 fts_hosts = rse_attrs[dest_rse_id].get('fts', None)
                 if not fts_hosts:
-                    logging.error('Source RSE %s FTS attribute not defined - SKIP REQUEST %s' % (rse, req_id))
+                    logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (dest_rse, req_id))
                     continue
                 if retry_count is None:
                     retry_count = 0


Test patch:
Repository: rucio/rucio

Issue:
Last python3 fixes for clients
Motivation
----------
Fix the last small changes to make the clients python3 compatible. This closes #819 
- didclient
- fileclient
- dq2client
- downloadclient


Modification
------------





Discussion:
You can ignore dq2client. It is deprecated and will be removed in less than 6 months.

Resolving patch:
diff --git a/lib/rucio/client/didclient.py b/lib/rucio/client/didclient.py
--- a/lib/rucio/client/didclient.py
+++ b/lib/rucio/client/didclient.py
@@ -25,6 +25,10 @@
 # - Eric Vaandering <ericvaandering@gmail.com>, 2018
 # - Asket Agarwal <asket.agarwal96@gmail.com>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+#
+# PY3K COMPATIBLE
+
+from __future__ import print_function
 
 try:
     from urllib import quote_plus
diff --git a/lib/rucio/client/fileclient.py b/lib/rucio/client/fileclient.py
--- a/lib/rucio/client/fileclient.py
+++ b/lib/rucio/client/fileclient.py
@@ -18,10 +18,16 @@
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2015
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+#
+# PY3K COMPATIBLE
 
 from __future__ import print_function
 
-from urllib import quote_plus
+try:
+    from urllib import quote_plus
+except ImportError:
+    from urllib.parse import quote_plus
 from json import loads
 from requests.status_codes import codes
 


Test patch:
Repository: rucio/rucio

Issue:
Make undertaker expire_rules size configurable
Motivation
----------
Currently the undertaker expires rules bigger than 10k locks, instead of deleting them immediately. This value should be configurable via the config table.





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -45,7 +45,7 @@
 from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.utils import str_to_date, is_archive, chunks
-from rucio.core import account_counter, rse_counter
+from rucio.core import account_counter, rse_counter, config as config_core
 from rucio.core.message import add_message
 from rucio.core.monitor import record_timer_block, record_counter
 from rucio.core.naming_convention import validate_name
@@ -635,7 +635,7 @@ def delete_dids(dids, account, expire_rules=False, session=None):
                     purge_replicas = False
                 else:
                     purge_replicas = True
-                if expire_rules and locks_ok_cnt + locks_replicating_cnt + locks_stuck_cnt > 10000:
+                if expire_rules and locks_ok_cnt + locks_replicating_cnt + locks_stuck_cnt > int(config_core.get('undertaker', 'expire_rules_locks_size', default=10000, session=session)):
                     # Expire the rule (soft=True)
                     rucio.core.rule.delete_rule(rule_id=rule_id, purge_replicas=purge_replicas, soft=True, delete_parent=True, nowait=True, session=session)
                     # Update expiration of did


Test patch:
Repository: rucio/rucio

Issue:
stager_area RSEs don't work with a non-deterministic source RSE
Motivation
----------

If the source RSE is non-deterministic then even if the deterministic flag is set to false on an associated staging_area the staged replicas don't set the path, making them unusable.

Modification
------------

Set the path when the RSE is non-deterministic.




Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/conveyor/finisher.py b/lib/rucio/daemons/conveyor/finisher.py
--- a/lib/rucio/daemons/conveyor/finisher.py
+++ b/lib/rucio/daemons/conveyor/finisher.py
@@ -219,7 +219,7 @@ def __handle_requests(reqs, suspicious_patterns, prepend_str=''):
                 replica['archived'] = False
 
                 # for TAPE, replica path is needed
-                if req['request_type'] == RequestType.TRANSFER and req['dest_rse_id'] in undeterministic_rses:
+                if req['request_type'] in (RequestType.TRANSFER, RequestType.STAGEIN) and req['dest_rse_id'] in undeterministic_rses:
                     if req['dest_rse_id'] not in rses_info:
                         dest_rse = get_rse_name(rse_id=req['dest_rse_id'])
                         rses_info[req['dest_rse_id']] = rsemanager.get_rse_info(dest_rse)


Test patch:
Repository: rucio/rucio

Issue:
Adding a rule for a staging area without a lifetime causes an internal server error
Motivation
----------

```
$ rucio add-rule protodune-sp:np04_raw_run_number_6300 1 FNAL_DCACHE_STAGING
2019-01-11 13:42:06,597	ERROR	An unknown exception occurred.
Details: no error information passed (http status code: 500 ('internal_server_error', 'server_error', '/o\\', '\xe2\x9c\x97'))
```



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -123,7 +123,7 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
 
             if lifetime is None:  # Check if one of the rses is a staging area
                 if [rse for rse in rses if rse.get('staging_area', False)]:
-                    raise StagingAreaRuleRequiresLifetime()
+                    raise StagingAreaRuleRequiresLifetime('Rules for a staging area must include a lifetime')
 
             # Check SCRATCHDISK Policy
             try:


Test patch:
Repository: rucio/rucio

Issue:
Decrease chunksize in add_constituents_to_archive from 50 to 20
Motivation
----------
Chunksize of 50 seems to lead to weird ORA error.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -311,7 +311,7 @@ def __add_files_to_archive(scope, name, files, account, ignore_duplicate=False,
     try:
         new_files and session.bulk_insert_mappings(models.DataIdentifier, new_files)
         if existing_files_condition:
-            for chunk in chunks(existing_files_condition, 50):
+            for chunk in chunks(existing_files_condition, 20):
                 session.query(models.DataIdentifier).\
                     with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
                     filter(models.DataIdentifier.did_type == DIDType.FILE).\


Test patch:
Repository: rucio/rucio

Issue:
Injector can crash when split_container option is enabled
Motivation
----------
The `split_container` option is enabled when requests on containers are done via R2D2. It creates rules on the constituent datasets of the container instead of a single rule on the container. 
- It can trigger `IntegrityError` in the case a rule for scope, name, account, rse_expression already exists. 
- Similarly the same error can be raised if 2 constituents contain the same datasets.

Modification
------------
To handle the first issue, the injector must check if a rule on scope, name, account, rse_expression already exists. To handle the second one, it should make a set from the list returned by `list_child_datasets`.




Discussion:
The check if a rule exists (uniqueness) is already done. The missing part is the uniqueness of list_child_datasets

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -605,6 +605,9 @@ def inject_rule(rule_id, session=None):
         logging.debug("Creating dataset rules for Split Container rule %s" % (str(rule.id)))
         # Get all child datasets and put rules on them
         dids = [{'scope': dataset['scope'], 'name': dataset['name']} for dataset in rucio.core.did.list_child_datasets(scope=rule.scope, name=rule.name, session=session)]
+        # Remove duplicates from the list of dictionaries
+        dids = [dict(t) for t in {tuple(d.items()) for d in dids}]
+        # Remove dids which already have a similar rule
         dids = [did for did in dids if session.query(models.ReplicationRule).filter_by(scope=did['scope'], name=did['name'], account=rule.account, rse_expression=rule.rse_expression).count() == 0]
         if rule.expires_at:
             lifetime = (rule.expires_at - datetime.utcnow()).days * 24 * 3600 + (rule.expires_at - datetime.utcnow()).seconds


Test patch:
Repository: rucio/rucio

Issue:
Documentation build fails
Motivation
----------
Currently the documentation builds fail with 

```
Running Sphinx v1.6.5
making output directory...
loading translations [en]... done

Traceback (most recent call last):
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/cmdline.py", line 305, in main
    opts.warningiserror, opts.tags, opts.verbosity, opts.jobs)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/application.py", line 196, in __init__
    self.setup_extension(extension)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/application.py", line 456, in setup_extension
    self.registry.load_extension(self, extname)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/registry.py", line 199, in load_extension
    raise ExtensionError(__('Could not import extension %s') % extname, err)
ExtensionError: Could not import extension rucio.common.doc.argparse.ext (exception: No module named rucio.common.doc.argparse.ext)

Extension error:
Could not import extension rucio.common.doc.argparse.ext (exception: No module named rucio.common.doc.argparse.ext)
```

I did not look too much into the issue yet, but I wonder why we added this `rucio.common.doc.argparse.ext` to the repository instead of using `sphinx-argparse`? @vingar do you maybe remember?


Discussion:

Resolving patch:
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -18,10 +18,10 @@
 #
 import os
 import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
 from mock import Mock as MagicMock
 
+sys.path.insert(len(sys.path) - 1, os.path.abspath('.'))
+
 
 class Mock(MagicMock):
     @classmethod


Test patch:
Repository: rucio/rucio

Issue:
Documentation build fails
Motivation
----------
Currently the documentation builds fail with 

```
Running Sphinx v1.6.5
making output directory...
loading translations [en]... done

Traceback (most recent call last):
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/cmdline.py", line 305, in main
    opts.warningiserror, opts.tags, opts.verbosity, opts.jobs)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/application.py", line 196, in __init__
    self.setup_extension(extension)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/application.py", line 456, in setup_extension
    self.registry.load_extension(self, extname)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/registry.py", line 199, in load_extension
    raise ExtensionError(__('Could not import extension %s') % extname, err)
ExtensionError: Could not import extension rucio.common.doc.argparse.ext (exception: No module named rucio.common.doc.argparse.ext)

Extension error:
Could not import extension rucio.common.doc.argparse.ext (exception: No module named rucio.common.doc.argparse.ext)
```

I did not look too much into the issue yet, but I wonder why we added this `rucio.common.doc.argparse.ext` to the repository instead of using `sphinx-argparse`? @vingar do you maybe remember?


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -22,7 +22,7 @@
 # - Wen Guan <wguan.icedew@gmail.com>, 2015-2016
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2018
-# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
@@ -128,7 +128,7 @@ def rse_exists(rse, session=None):
 @read_session
 def sort_rses(rses, session=None):
     """
-    Sort a list of RSES by srm free space (ascending order).
+    Sort a list of RSES by free space (ascending order).
 
     :param rses: List of RSEs.
     :param session: The database session in use.
@@ -143,7 +143,7 @@ def sort_rses(rses, session=None):
 
     false_value = False
     query = session.query(models.RSE.rse, models.RSE.staging_area, models.RSEUsage.rse_id).\
-        filter(or_(models.RSEUsage.source == 'srm', models.RSEUsage.source == 'gsiftp')).\
+        filter(models.RSEUsage.source == 'storage').\
         filter(models.RSEUsage.rse_id == models.RSE.id).\
         filter(models.RSE.deleted == false_value)
     condition = []


Test patch:
Repository: rucio/rucio

Issue:
Documentation build fails
Motivation
----------
Currently the documentation builds fail with 

```
Running Sphinx v1.6.5
making output directory...
loading translations [en]... done

Traceback (most recent call last):
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/cmdline.py", line 305, in main
    opts.warningiserror, opts.tags, opts.verbosity, opts.jobs)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/application.py", line 196, in __init__
    self.setup_extension(extension)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/application.py", line 456, in setup_extension
    self.registry.load_extension(self, extname)
  File "/home/docs/checkouts/readthedocs.org/user_builds/rucio/envs/latest/local/lib/python2.7/site-packages/sphinx/registry.py", line 199, in load_extension
    raise ExtensionError(__('Could not import extension %s') % extname, err)
ExtensionError: Could not import extension rucio.common.doc.argparse.ext (exception: No module named rucio.common.doc.argparse.ext)

Extension error:
Could not import extension rucio.common.doc.argparse.ext (exception: No module named rucio.common.doc.argparse.ext)
```

I did not look too much into the issue yet, but I wonder why we added this `rucio.common.doc.argparse.ext` to the repository instead of using `sphinx-argparse`? @vingar do you maybe remember?


Discussion:

Resolving patch:
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -20,7 +20,7 @@
 import sys
 from mock import Mock as MagicMock
 
-sys.path.insert(len(sys.path) - 1, os.path.abspath('.'))
+sys.path.insert(len(sys.path), os.path.abspath('../../lib'))
 
 
 class Mock(MagicMock):


Test patch:
Repository: rucio/rucio

Issue:
View for suspicious files
Motivation
----------
The sites/clouds squads need a view to list the suspicious files on their site.





Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/protocols/gfal.py b/lib/rucio/rse/protocols/gfal.py
--- a/lib/rucio/rse/protocols/gfal.py
+++ b/lib/rucio/rse/protocols/gfal.py
@@ -18,7 +18,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2016
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2016-2017
 # - Tobias Wegner <twegner@cern.ch>, 2017
-# - Nicolo Magini <Nicolo.Magini@cern.ch>, 2018
+# - Nicolo Magini <Nicolo.Magini@cern.ch>, 2018-2019
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
@@ -35,6 +35,9 @@
 except ImportError:
     # PY3
     import urllib.parse as urlparse
+
+from threading import Timer
+
 from rucio.common import exception, config
 from rucio.common.constraints import STRING_TYPES
 from rucio.rse.protocols import protocol
@@ -356,6 +359,15 @@ def stat(self, path):
 
         return ret
 
+    def __gfal2_cancel(self):
+        """
+        Cancel all gfal operations in progress.
+        """
+
+        ctx = self.__ctx
+        if ctx:
+            ctx.cancel()
+
     def __gfal2_copy(self, src, dest, src_spacetoken=None, dest_spacetoken=None, transfer_timeout=None):
         """
         Uses gfal2 to copy file from src to dest.
@@ -380,6 +392,7 @@ def __gfal2_copy(self, src, dest, src_spacetoken=None, dest_spacetoken=None, tra
             params.dst_spacetoken = str(dest_spacetoken)
         if transfer_timeout:
             params.timeout = int(transfer_timeout)
+            watchdog = Timer(params.timeout + 60, self.__gfal2_cancel)
 
         dir_name = os.path.dirname(dest)
         # This function will be removed soon. gfal2 will create parent dir automatically.
@@ -389,9 +402,15 @@ def __gfal2_copy(self, src, dest, src_spacetoken=None, dest_spacetoken=None, tra
             pass
 
         try:
+            if transfer_timeout:
+                watchdog.start()
             ret = ctx.filecopy(params, str(src), str(dest))
+            if transfer_timeout:
+                watchdog.cancel()
             return ret
         except gfal2.GError as error:  # pylint: disable=no-member
+            if transfer_timeout:
+                watchdog.cancel()
             if error.code == errno.ENOENT or 'No such file' in str(error):
                 raise exception.SourceNotFound(error)
             raise exception.RucioException(error)


Test patch: