internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: rucio/rucio

Issue:
Hermes2: Submission of deletion events in local time
Motivation
----------
Hermes2 submits the deletion events in local time. It needs to be changed to UTC.




Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/hermes/hermes2.py b/lib/rucio/daemons/hermes/hermes2.py
--- a/lib/rucio/daemons/hermes/hermes2.py
+++ b/lib/rucio/daemons/hermes/hermes2.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2020
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2020-2021
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 
@@ -354,8 +354,8 @@ def aggregate_to_influx(messages, bin_size, endpoint, prepend_str):
         elif event_type in ['deletion-failed', 'deletion-done']:
             created_at = message['created_at']
             if bin_size == '1m':
-                created_at = created_at.replace(second=0, microsecond=0)
-            created_at = int(created_at.strftime('%s')) * 1000000000
+                created_at = created_at.replace(second=0, microsecond=0, tzinfo=datetime.timezone.utc).timestamp()
+            created_at = int(created_at) * 1000000000
             created_at += microsecond
             if created_at not in bins:
                 bins[created_at] = {}


Test patch:
Repository: rucio/rucio

Issue:
core/rule.py reencodes meta over and over again
Motivation
----------

core/replica.py reencodes meta to JSON over and over again inside a loop on DIDs. Eventually this makes a string so big it causes an exception in oracle. It's not clear if this also corrupts the intended metadata

Modification
------------

Use a dedicated variable for the results of the JSON encoding




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -30,7 +30,7 @@
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
-# - Eric Vaandering <ewv@fnal.gov>, 2020
+# - Eric Vaandering <ewv@fnal.gov>, 2020-2021
 
 from __future__ import division
 
@@ -231,11 +231,12 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
 
             # 4. Create the replication rule
             with record_timer_block('rule.add_rule.create_rule'):
+                meta_json = None
                 if meta is not None:
                     try:
-                        meta = json.dumps(meta)
+                        meta_json = json.dumps(meta)
                     except Exception:
-                        meta = None
+                        meta_json = None
 
                 new_rule = models.ReplicationRule(account=account,
                                                   name=elem['name'],
@@ -257,7 +258,7 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
                                                   ignore_account_limit=ignore_account_limit,
                                                   priority=priority,
                                                   split_container=split_container,
-                                                  meta=meta,
+                                                  meta=meta_json,
                                                   eol_at=eol_at)
                 try:
                     new_rule.save(session=session)


Test patch:
Repository: rucio/rucio

Issue:
imp / importlib incompatibility
Motivation
----------

The fallback load of `imp` is not good enough, this would only start working on Python 3.9.


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py
--- a/lib/rucio/client/baseclient.py
+++ b/lib/rucio/client/baseclient.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2012-2020 CERN
+# Copyright 2012-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2020
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2018
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2020
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2021
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2015
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2015-2018
@@ -46,7 +46,14 @@
     import importlib
     importlib.util.find_spec('')
 except AttributeError:
+    pass
+
+# Fallback until Python <3.9
+# Must be removed with Python 3.9
+try:
     import imp
+except AttributeError:
+    pass
 
 import os
 import random
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -39,7 +39,14 @@
     import importlib
     importlib.util.find_spec('')
 except AttributeError:
+    pass
+
+# Fallback until Python <3.9
+# Must be removed with Python 3.9
+try:
     import imp
+except AttributeError:
+    pass
 
 import base64
 import copy


Test patch:
Repository: rucio/rucio

Issue:
imp deprecation warning in favour of importlib
Motivation
----------
imp was deprecated long back in favour of importlib with python 3.3. 
Since Python 3.9 will be the last version providing backwards python2 compatibility, these libraries may cause an error. 

![long](https://user-images.githubusercontent.com/69708588/109564260-d516e600-7b06-11eb-8325-d2d3d3805d9c.JPG)

Modification
------------
Implement dynamic loading using importlib, try to ensure backward compatibility for the time being



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -39,10 +39,10 @@
 
 import copy
 import datetime
-import imp
 import json
 import logging
 import re
+import sys
 import time
 from typing import TYPE_CHECKING
 
@@ -82,11 +82,21 @@
 EXTRA_MODULES = {'globus_sdk': False}
 
 for extra_module in EXTRA_MODULES:
-    try:
-        imp.find_module(extra_module)
-        EXTRA_MODULES[extra_module] = True
-    except ImportError:
-        EXTRA_MODULES[extra_module] = False
+    if sys.version_info < (3, 5):
+        try:
+            import imp
+            imp.find_module(extra_module)
+            EXTRA_MODULES[extra_module] = True
+        except ImportError:
+            EXTRA_MODULES[extra_module] = False
+
+    else:
+        try:
+            import importlib
+            importlib.util.find_spec(extra_module)
+            EXTRA_MODULES[extra_module] = True
+        except ModuleNotFoundError:
+            EXTRA_MODULES[extra_module] = False
 
 if EXTRA_MODULES['globus_sdk']:
     from rucio.transfertool.globus import GlobusTransferTool  # pylint: disable=import-error


Test patch:
Repository: rucio/rucio

Issue:
The Dark Reaper should run continuously instead of exiting early when there’s nothing to do
Motivation
----------

The Dark Reaper has its own implementation of `list_rses()` which only returns RSEs that have at least one quarantined replica. This is not optimal for two reasons:

 1.  If there are no quarantined replicas, then the Reaper exists early. Daemons running on Kubernetes are expected to run continuously.
 2. If quarantined replicas are added for an RSE that didn’t have any when the daemon started, then they won’t be processed until the next restart.

Modification
------------

Replace the custom `list_rses()` implementation with the one from the core.


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/reaper/dark_reaper.py b/lib/rucio/daemons/reaper/dark_reaper.py
--- a/lib/rucio/daemons/reaper/dark_reaper.py
+++ b/lib/rucio/daemons/reaper/dark_reaper.py
@@ -52,6 +52,7 @@
 from rucio.core.quarantined_replica import (list_quarantined_replicas,
                                             delete_quarantined_replicas,
                                             list_rses)
+import rucio.core.rse as rse_core
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.vo import list_vos
 from rucio.rse import rsemanager as rsemgr
@@ -90,6 +91,7 @@ def reaper(rses=[], worker_number=0, total_workers=1, chunk_size=100, once=False
                          .format(locals()))
             nothing_to_do = True
 
+            rses = list(set(rses) & set(list_rses()))
             random.shuffle(rses)
             for rse_id in rses:
                 replicas = list_quarantined_replicas(rse_id=rse_id,
@@ -232,7 +234,7 @@ def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None,
 
     all_rses = []
     for vo in vos:
-        all_rses.extend(list_rses(filters={'vo': vo}))
+        all_rses.extend([rse['id'] for rse in rse_core.list_rses(filters={'vo': vo})])
 
     if rses:
         invalid = set(rses) - set([rse['rse'] for rse in all_rses])


Test patch:
Repository: rucio/rucio

Issue:
Transfers without sources stall the conveyor submitter
Motivation
----------
Transfers without sources are never removed from the transfer queue and stall the submitter.

Modification
------------
```
if req_id not in reqs_no_source:
    reqs_no_source.append(req_id)
```
needs to be added before https://github.com/rucio/rucio/blob/510adbd9be99c7bc3111cc88a68fb889b222dd40/lib/rucio/core/transfer.py#L425


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -6,7 +6,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Authors:
-# - Martin Barisits, <martin.barisits@cern.ch>, 2017-2018
+# - Martin Barisits, <martin.barisits@cern.ch>, 2017-2021
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2017-2018
 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2018-2019
 # - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
@@ -422,6 +422,8 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
 
         # source_rse_id will be None if no source replicas
         # rse will be None if rse is staging area
+        if req_id not in reqs_no_source:
+            reqs_no_source.append(req_id)
         if source_rse_id is None or rse is None:
             continue
 


Test patch:
Repository: rucio/rucio

Issue:
Fix Docstring in core/quarantined_replica.py
Motivation
----------
The current docstring lists two extra parameters, namely files and ignore_availability.
It does not include one parameter, namely replicas.


Modification
------------
Add docstring for missing parameter and remove for the additional parameter.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/quarantined_replica.py b/lib/rucio/core/quarantined_replica.py
--- a/lib/rucio/core/quarantined_replica.py
+++ b/lib/rucio/core/quarantined_replica.py
@@ -82,8 +82,7 @@ def delete_quarantined_replicas(rse_id, replicas, session=None):
     Delete file replicas.
 
     :param rse_id: the rse id.
-    :param files: the list of files to delete.
-    :param ignore_availability: Ignore the RSE blacklisting.
+    :param replicas: A list of dicts with the replica information.
     :param session: The database session in use.
     """
 


Test patch:
Repository: rucio/rucio

Issue:
Rule page shows RSE IDs instead of names
![image](https://user-images.githubusercontent.com/2452306/63096904-621f3c00-bf6f-11e9-8608-ad812655c18e.png)


Discussion:
Hello, I was thinking that I can try to work on this issue. Can this be assigned to me?

Hi @DhruvSondhi please go ahead with the implementation :-)

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -2248,11 +2248,11 @@ def examine_rule(rule_id, session=None):
                     last_request = transfers[0]
                     last_error = last_request.state
                     last_time = last_request.created_at
-                    last_source = None if last_request.source_rse_id is None else last_request.source_rse_id
+                    last_source = None if last_request.source_rse_id is None else get_rse_name(rse_id=last_request.source_rse_id, session=session)
                     available_replicas = session.query(models.RSEFileAssociation).filter_by(scope=lock.scope, name=lock.name, state=ReplicaState.AVAILABLE).all()
 
                     for replica in available_replicas:
-                        sources.append((replica.rse_id,
+                        sources.append((get_rse_name(rse_id=replica.rse_id, session=session),
                                         True if get_rse(rse_id=replica.rse_id, session=session).availability >= 4 else False))
 
                 result['transfers'].append({'scope': lock.scope,


Test patch:
Repository: rucio/rucio

Issue:
list-dids-extended with new metadata plugin currently fails
Motivation
----------
```
rucio set-metadata --did
CTA_LAPP_FREDERIC:cta_data_DL0_ObsId2021_02_22_15_49_11_975413919_TelId0.h
df5 --key ObsID --value 1

rucio get-metadata
CTA_LAPP_FREDERIC:cta_data_DL0_ObsId2021_02_22_15_49_11_975413919_TelId0.h
df5 --plugin ALL
bsID:           1
access_cnt:      None
accessed_at:     None
....

rucio list-dids-extended --filter ObsID=1 CTA_LAPP_FREDERIC:*
/usr/lib/python2.7/site-packages/paramiko/transport.py:33:
CryptographyDeprecationWarning: Python 2 is no longer supported by the
Python core team. Support for it is now deprecated in cryptography, and
will be removed in a future release.
  from cryptography.hazmat.backends import default_backend
2021-02-22 15:58:35,693 ERROR   An unknown exception occurred.
Details: no error information passed (http status code: 500
('internal_server_error', 'server_error', '/o\\', '\xe2\x9c\x97'))
```


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did_meta_plugins/json_meta.py b/lib/rucio/core/did_meta_plugins/json_meta.py
--- a/lib/rucio/core/did_meta_plugins/json_meta.py
+++ b/lib/rucio/core/did_meta_plugins/json_meta.py
@@ -30,6 +30,7 @@
 # - Brandon White, <bjwhite@fnal.gov>, 2019
 # - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Rizart Dona <rizart.dona@cern.ch>, 2021
 
 import json as json_lib
 
@@ -160,7 +161,7 @@ def list_dids(self, scope, filters, type='collection', ignore_case=False, limit=
         filters.pop('name', None)
         for k, v in iteritems(filters):
             if session.bind.dialect.name == 'oracle':
-                query = query.filter(text("json_exists(meta,'$.%s?(@==''%s'')')" % (k, v)))
+                query = query.filter(text("json_exists(meta,'$?(@.{} == \"{}\")')".format(k, v)))
             else:
                 query = query.filter(cast(models.DidMeta.meta[k], String) == type_coerce(v, JSON))
 


Test patch:
Repository: rucio/rucio

Issue:
Allow updating rule comment
Motivation
----------
It would be nice to also allow updating the `comment` field in the `update_replication_rule` API.


Modification
------------
Probably just need to add `comment` to the list in https://github.com/rucio/rucio/blob/9ab74408d790641d451b9b4fccb98f5928685a6d/lib/rucio/core/rule.py#L1281





Discussion:
As an aside, it seems in `add_replication_rule` it is “comment” but in the dictionary returned from `list_replication_rules` it is “comments”

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -378,7 +378,8 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
             insert_rule_history(rule=new_rule, recent=True, longterm=True, session=session)
 
             if use_new_rule_algorithm:
-                logger(logging.INFO, "Created rule %s [%d/%d/%d] with new algorithm for did %s:%s in state %s", str(new_rule.id), new_rule.locks_ok_cnt, new_rule.locks_replicating_cnt, new_rule.locks_stuck_cnt, new_rule.scope, new_rule.name, str(new_rule.state))
+                logger(logging.INFO, "Created rule %s [%d/%d/%d] with new algorithm for did %s:%s in state %s", str(new_rule.id), new_rule.locks_ok_cnt,
+                       new_rule.locks_replicating_cnt, new_rule.locks_stuck_cnt, new_rule.scope, new_rule.name, str(new_rule.state))
             else:
                 logger(logging.INFO, "Created rule %s [%d/%d/%d] for did %s:%s in state %s", str(new_rule.id), new_rule.locks_ok_cnt, new_rule.locks_replicating_cnt, new_rule.locks_stuck_cnt, new_rule.scope, new_rule.name, str(new_rule.state))
 
@@ -1282,7 +1283,7 @@ def update_rule(rule_id, options, session=None):
     :raises:            RuleNotFound if no Rule can be found, InputValidationError if invalid option is used, ScratchDiskLifetimeConflict if wrong ScratchDiskLifetime is used.
     """
 
-    valid_options = ['locked', 'lifetime', 'account', 'state', 'activity', 'source_replica_expression', 'cancel_requests', 'priority', 'child_rule_id', 'eol_at', 'meta', 'purge_replicas']
+    valid_options = ['comment', 'locked', 'lifetime', 'account', 'state', 'activity', 'source_replica_expression', 'cancel_requests', 'priority', 'child_rule_id', 'eol_at', 'meta', 'purge_replicas']
 
     for key in options:
         if key not in valid_options:
@@ -1303,6 +1304,9 @@ def update_rule(rule_id, options, session=None):
             if key == 'source_replica_expression':
                 rule.source_replica_expression = options['source_replica_expression']
 
+            if key == 'comment':
+                rule.comments = options['comment']
+
             if key == 'activity':
                 validate_schema('activity', options['activity'], vo=rule.account.vo)
                 rule.activity = options['activity']


Test patch:
Repository: rucio/rucio

Issue:
Update PyYAML for tests due to CVE-2020-14343
Motivation
----------
https://github.com/advisories/GHSA-8q59-q68h-6hv4


Modification
------------
Update PyYAML for tests to 5.4


Discussion:

Resolving patch:
diff --git a/setup_rucio.py b/setup_rucio.py
--- a/setup_rucio.py
+++ b/setup_rucio.py
@@ -166,7 +166,7 @@ def write_requirements():
 postgresql_extras = ['psycopg2-binary==2.8.6']
 mysql_extras = ['PyMySQL']
 kerberos_extras = ['kerberos>=1.3.0', 'pykerberos>=1.2.1', 'requests-kerberos>=0.12.0']
-globus_extras = ['PyYAML==5.1.1', 'globus-sdk==1.8.0']
+globus_extras = ['PyYAML==5.4', 'globus-sdk==1.8.0']
 saml_extras = ['python3-saml>=1.6.0']
 dev_extras = parse_requirements(requirements_files=['etc/pip-requires-test', ])
 requires = parse_requirements(requirements_files=requirements_files)


Test patch:
Repository: rucio/rucio

Issue:
Atropos can’t fetch the list of rules beyond EOL
Motivation
----------

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/atropos/atropos.py", line 110, in atropos
    rses = parse_expression(rule.rse_expression, filter={'vo': rule.account.vo})
AttributeError: 'result' object has no attribute 'account'
```

This bug was introduced with commit ec57f5731f.

Modification
------------

Simply add `account` to the returned columns of `get_rules_beyond_eol()`.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -1628,7 +1628,8 @@ def get_rules_beyond_eol(date_check, worker_number, total_workers, session):
                           models.ReplicationRule.locked,
                           models.ReplicationRule.id,
                           models.ReplicationRule.eol_at,
-                          models.ReplicationRule.expires_at).\
+                          models.ReplicationRule.expires_at,
+                          models.ReplicationRule.account).\
         filter(models.ReplicationRule.eol_at < date_check)
 
     query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')


Test patch:
Repository: rucio/rucio

Issue:
Account RSE Usage page occasionally does not work
This appears to happen when the requests done by the Web UI are processed by a backend running on Kubernetes, which use Flask instead of web.py. Making a request to https://rucio-ui.cern.ch/proxy/accounts/ddmadmin/usage/ returns a 404 error.

![image](https://user-images.githubusercontent.com/2452306/114176857-02dc1f80-993c-11eb-8bf8-f7ee86381852.png)



Discussion:

Resolving patch:
diff --git a/lib/rucio/web/rest/flaskapi/v1/accounts.py b/lib/rucio/web/rest/flaskapi/v1/accounts.py
--- a/lib/rucio/web/rest/flaskapi/v1/accounts.py
+++ b/lib/rucio/web/rest/flaskapi/v1/accounts.py
@@ -597,6 +597,10 @@ def blueprint(no_doc=True):
     usage_view = LocalUsage.as_view('usage')
     bp.add_url_rule('/<account>/usage/local', view_func=usage_view, methods=['get', ])
     bp.add_url_rule('/<account>/usage', view_func=usage_view, methods=['get', ])
+    if no_doc:
+        # for backwards-compatibility
+        # rule without trailing slash needs to be added before rule with trailing slash
+        bp.add_url_rule('/<account>/usage/', view_func=usage_view, methods=['get', ])
     bp.add_url_rule('/<account>/usage/local/<rse>', view_func=usage_view, methods=['get', ])
     bp.add_url_rule('/<account>/usage/<rse>', view_func=usage_view, methods=['get', ])
     global_usage_view = GlobalUsage.as_view('global_usage')
diff --git a/lib/rucio/web/rest/flaskapi/v1/replicas.py b/lib/rucio/web/rest/flaskapi/v1/replicas.py
--- a/lib/rucio/web/rest/flaskapi/v1/replicas.py
+++ b/lib/rucio/web/rest/flaskapi/v1/replicas.py
@@ -889,7 +889,7 @@ def blueprint(no_doc=True):
         bp.add_url_rule('', view_func=replicas_view, methods=['post', 'put', 'delete'])
     bp.add_url_rule('/', view_func=replicas_view, methods=['post', 'put', 'delete'])
     suspicious_replicas_view = SuspiciousReplicas.as_view('suspicious_replicas')
-    bp.add_url_rule('/suspicious', view_func=suspicious_replicas_view, methods=['post', ])
+    bp.add_url_rule('/suspicious', view_func=suspicious_replicas_view, methods=['get', 'post'])
     bad_replicas_states_view = BadReplicasStates.as_view('bad_replicas_states')
     bp.add_url_rule('/bad/states', view_func=bad_replicas_states_view, methods=['get', ])
     bad_replicas_summary_view = BadReplicasSummary.as_view('bad_replicas_summary')
@@ -915,7 +915,7 @@ def blueprint(no_doc=True):
 
     if no_doc:
         bp.add_url_rule('/list/', view_func=list_replicas_view, methods=['post', ])
-        bp.add_url_rule('/suspicious/', view_func=suspicious_replicas_view, methods=['post', ])
+        bp.add_url_rule('/suspicious/', view_func=suspicious_replicas_view, methods=['get', 'post'])
         bp.add_url_rule('/bad/states/', view_func=bad_replicas_states_view, methods=['get', ])
         bp.add_url_rule('/bad/summary/', view_func=bad_replicas_summary_view, methods=['get', ])
         bp.add_url_rule('/bad/pfns/', view_func=bad_replicas_pfn_view, methods=['post', ])


Test patch: