internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: rucio/rucio

Issue:
Make limit variable of the judge-evaluator configurable
Motivation
----------
Currently the judge-evaluator has a hard-coded 100 did limit.


Modification
------------
Make it configurable.


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/judge/evaluator.py b/lib/rucio/daemons/judge/evaluator.py
--- a/lib/rucio/daemons/judge/evaluator.py
+++ b/lib/rucio/daemons/judge/evaluator.py
@@ -55,7 +55,7 @@
 graceful_stop = threading.Event()
 
 
-def re_evaluator(once=False, sleep_time=30):
+def re_evaluator(once=False, sleep_time=30, did_limit=100):
     """
     Main loop to check the re-evaluation of dids.
     """
@@ -84,7 +84,7 @@ def re_evaluator(once=False, sleep_time=30):
             # Select a bunch of dids for re evaluation for this worker
             dids = get_updated_dids(total_workers=heartbeat['nr_threads'],
                                     worker_number=heartbeat['assign_thread'],
-                                    limit=100,
+                                    limit=did_limit,
                                     blocked_dids=[(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids])
             logging.debug('re_evaluator[%s/%s] index query time %f fetch size is %d (%d blocked)' % (heartbeat['assign_thread'],
                                                                                                      heartbeat['nr_threads'],
@@ -172,7 +172,7 @@ def stop(signum=None, frame=None):
     graceful_stop.set()
 
 
-def run(once=False, threads=1, sleep_time=30):
+def run(once=False, threads=1, sleep_time=30, did_limit=100):
     """
     Starts up the Judge-Eval threads.
     """
@@ -186,11 +186,12 @@ def run(once=False, threads=1, sleep_time=30):
     sanity_check(executable=executable, hostname=hostname)
 
     if once:
-        re_evaluator(once)
+        re_evaluator(once=once, did_limit=did_limit)
     else:
         logging.info('Evaluator starting %s threads' % str(threads))
         threads = [threading.Thread(target=re_evaluator, kwargs={'once': once,
-                                                                 'sleep_time': sleep_time}) for i in range(0, threads)]
+                                                                 'sleep_time': sleep_time,
+                                                                 'did_limit': did_limit}) for i in range(0, threads)]
         [t.start() for t in threads]
         # Interruptible joins require a timeout.
         while threads[0].is_alive():


Test patch:
Repository: rucio/rucio

Issue:
Abacus : Add option to set limit
Motivation
----------
There's no possibility to control the size of the chunk in abacus







Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/abacus/collection_replica.py b/lib/rucio/daemons/abacus/collection_replica.py
--- a/lib/rucio/daemons/abacus/collection_replica.py
+++ b/lib/rucio/daemons/abacus/collection_replica.py
@@ -101,7 +101,7 @@ def stop(signum=None, frame=None):
     graceful_stop.set()
 
 
-def run(once=False, threads=1, sleep_time=10):
+def run(once=False, threads=1, sleep_time=10, limit=1000):
     """
     Starts up the Abacus-Collection-Replica threads.
     """
@@ -119,8 +119,8 @@ def run(once=False, threads=1, sleep_time=10):
         collection_replica_update(once)
     else:
         logging.info('main: starting threads')
-        threads = [threading.Thread(target=collection_replica_update, kwargs={'once': once, 'sleep_time': sleep_time})
-                   for i in range(0, threads)]
+        threads = [threading.Thread(target=collection_replica_update, kwargs={'once': once, 'sleep_time': sleep_time, 'limit': limit})
+                   for _ in range(0, threads)]
         [t.start() for t in threads]
         logging.info('main: waiting for interrupts')
         # Interruptible joins require a timeout.


Test patch:
Repository: rucio/rucio

Issue:
Python 2 compatibility broke with 1.26 release
Motivation
----------
For some reason (probably due to a testing dependency) this did not show up in our tests:

```
$ rucio --version
Traceback (most recent call last):
  File "/opt/rucio/bin/rucio", line 71, in <module>
    from rucio.client import Client
  File "/opt/rucio/lib/rucio/client/__init__.py", line 20, in <module>
    from .client import Client  # noqa: F401
  File "/opt/rucio/lib/rucio/client/client.py", line 30, in <module>
    from rucio.client.accountclient import AccountClient
  File "/opt/rucio/lib/rucio/client/accountclient.py", line 38, in <module>
    from rucio.client.baseclient import BaseClient
  File "/opt/rucio/lib/rucio/client/baseclient.py", line 69, in <module>
    from rucio.common.extra import import_extras
  File "/opt/rucio/lib/rucio/common/extra.py", line 20, in <module>
    from typing import TYPE_CHECKING
ImportError: No module named typing
```


Modification
------------
Remove any typing imports for 1.26, add dependency to `typing` for 1.27 for python < 3.5


Python 2 compatibility broke with 1.26 release
Motivation
----------
For some reason (probably due to a testing dependency) this did not show up in our tests:

```
$ rucio --version
Traceback (most recent call last):
  File "/opt/rucio/bin/rucio", line 71, in <module>
    from rucio.client import Client
  File "/opt/rucio/lib/rucio/client/__init__.py", line 20, in <module>
    from .client import Client  # noqa: F401
  File "/opt/rucio/lib/rucio/client/client.py", line 30, in <module>
    from rucio.client.accountclient import AccountClient
  File "/opt/rucio/lib/rucio/client/accountclient.py", line 38, in <module>
    from rucio.client.baseclient import BaseClient
  File "/opt/rucio/lib/rucio/client/baseclient.py", line 69, in <module>
    from rucio.common.extra import import_extras
  File "/opt/rucio/lib/rucio/common/extra.py", line 20, in <module>
    from typing import TYPE_CHECKING
ImportError: No module named typing
```


Modification
------------
Remove any typing imports for 1.26, add dependency to `typing` for 1.27 for python < 3.5




Discussion:

Resolving patch:
diff --git a/lib/rucio/common/extra.py b/lib/rucio/common/extra.py
--- a/lib/rucio/common/extra.py
+++ b/lib/rucio/common/extra.py
@@ -17,14 +17,9 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
 
 import importlib
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from typing import List, Dict, Any
 
 
 def import_extras(module_list):
-    # type: (List[str]) -> Dict[str, Any]
     out = dict()
     for mod in module_list:
         out[mod] = None


Test patch:
Repository: rucio/rucio

Issue:
Python 2 compatibility broke with 1.26 release
Motivation
----------
For some reason (probably due to a testing dependency) this did not show up in our tests:

```
$ rucio --version
Traceback (most recent call last):
  File "/opt/rucio/bin/rucio", line 71, in <module>
    from rucio.client import Client
  File "/opt/rucio/lib/rucio/client/__init__.py", line 20, in <module>
    from .client import Client  # noqa: F401
  File "/opt/rucio/lib/rucio/client/client.py", line 30, in <module>
    from rucio.client.accountclient import AccountClient
  File "/opt/rucio/lib/rucio/client/accountclient.py", line 38, in <module>
    from rucio.client.baseclient import BaseClient
  File "/opt/rucio/lib/rucio/client/baseclient.py", line 69, in <module>
    from rucio.common.extra import import_extras
  File "/opt/rucio/lib/rucio/common/extra.py", line 20, in <module>
    from typing import TYPE_CHECKING
ImportError: No module named typing
```


Modification
------------
Remove any typing imports for 1.26, add dependency to `typing` for 1.27 for python < 3.5


Python 2 compatibility broke with 1.26 release
Motivation
----------
For some reason (probably due to a testing dependency) this did not show up in our tests:

```
$ rucio --version
Traceback (most recent call last):
  File "/opt/rucio/bin/rucio", line 71, in <module>
    from rucio.client import Client
  File "/opt/rucio/lib/rucio/client/__init__.py", line 20, in <module>
    from .client import Client  # noqa: F401
  File "/opt/rucio/lib/rucio/client/client.py", line 30, in <module>
    from rucio.client.accountclient import AccountClient
  File "/opt/rucio/lib/rucio/client/accountclient.py", line 38, in <module>
    from rucio.client.baseclient import BaseClient
  File "/opt/rucio/lib/rucio/client/baseclient.py", line 69, in <module>
    from rucio.common.extra import import_extras
  File "/opt/rucio/lib/rucio/common/extra.py", line 20, in <module>
    from typing import TYPE_CHECKING
ImportError: No module named typing
```


Modification
------------
Remove any typing imports for 1.26, add dependency to `typing` for 1.27 for python < 3.5




Discussion:

Resolving patch:
diff --git a/setuputil.py b/setuputil.py
--- a/setuputil.py
+++ b/setuputil.py
@@ -34,6 +34,7 @@
         'tabulate',
         'six',
         'jsonschema',
+        'typing',
     ],
     'ssh': ['paramiko'],
     'kerberos': [


Test patch:
Repository: rucio/rucio

Issue:
Fix setup_webui script
Motivation
----------
Script has a wrong import, needs to be fixed.




Discussion:

Resolving patch:
diff --git a/setup_webui.py b/setup_webui.py
--- a/setup_webui.py
+++ b/setup_webui.py
@@ -35,7 +35,7 @@
     from setuputil import get_rucio_version
 
 name = 'rucio-webui'
-packages = ['rucio', 'rucio.web', 'rucio.web.ui', 'rucio.web.ui.flask', 'rucio.web.flask.common']
+packages = ['rucio', 'rucio.web', 'rucio.web.ui', 'rucio.web.ui.flask', 'rucio.web.ui.flask.common']
 data_files = []
 description = "Rucio WebUI Package"
 


Test patch:
Repository: rucio/rucio

Issue:
enum dependency issue for python2
Motivation
----------
Since we do require the python3 `enum` package in https://github.com/rucio/rucio/blob/master/lib/rucio/common/utils.py which is used by the python 2.7 clients we need to add the backport of it to the py27 clients package.

Modification
------------
Add https://pypi.org/project/enum34/ to python27 dependencies for clients.



Discussion:

Resolving patch:
diff --git a/setuputil.py b/setuputil.py
--- a/setuputil.py
+++ b/setuputil.py
@@ -15,6 +15,7 @@
 #
 # Authors:
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
+# - Martin Barisits <martin.barisits@cern.ch>, 2021
 
 from __future__ import print_function
 
@@ -35,6 +36,7 @@
         'six',
         'jsonschema',
         'typing',
+        'enum34',
     ],
     'ssh': ['paramiko'],
     'kerberos': [


Test patch:
Repository: rucio/rucio

Issue:
list-file-repicas --all-states should show the states
Motivation
----------

The WebUI shows this information. Without it, `--all-states` isn’t that helpful.

Modification
------------

Add an additional column when `--all-states` is used.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/constants.py b/lib/rucio/common/constants.py
--- a/lib/rucio/common/constants.py
+++ b/lib/rucio/common/constants.py
@@ -20,8 +20,10 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018-2021
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Sahan Dilshan <32576163+sahandilshan@users.noreply.github.com>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
 from collections import namedtuple
+from enum import Enum
 
 """
 Constants.
@@ -52,3 +54,13 @@
                                                   'CANCELED')
 
 FTS_COMPLETE_STATE = namedtuple('FTS_COMPLETE_STATE', ['OK', 'ERROR'])('Ok', 'Error')
+
+
+class ReplicaState(Enum):
+    # From rucio.db.sqla.constants, update that file at the same time than this
+    AVAILABLE = 'A'
+    UNAVAILABLE = 'U'
+    COPYING = 'C'
+    BEING_DELETED = 'B'
+    BAD = 'D'
+    TEMPORARY_UNAVAILABLE = 'T'
diff --git a/lib/rucio/db/sqla/constants.py b/lib/rucio/db/sqla/constants.py
--- a/lib/rucio/db/sqla/constants.py
+++ b/lib/rucio/db/sqla/constants.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015-2020 CERN
+# Copyright 2015-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,8 @@
 # - Jaroslav Guenther <jaroslav.guenther@cern.ch>, 2019
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Sahan Dilshan <32576163+sahandilshan@users.noreply.github.com>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
 from datetime import datetime
 from enum import Enum
@@ -116,6 +118,7 @@ class LockState(Enum):
 
 
 class ReplicaState(Enum):
+    # Update rucio.common.constants at the same time than this
     AVAILABLE = 'A'
     UNAVAILABLE = 'U'
     COPYING = 'C'


Test patch:
Repository: rucio/rucio

Issue:
Skip updating replica in uploadclient if register_after_upload is true
Motivation
----------
In https://github.com/rucio/rucio/blob/b6092bcd78f8cfac079e5b3cc25a92b4163960ed/lib/rucio/client/uploadclient.py#L304-L308 there is no need to update the replica state if it is registered after upload, since it will already be registered with state `A`. Further, the permissions for updating a replica can be (are for CMS) more restrictive than for injection.

Modification
------------
Put L306-309 in an `else` block. Also, `update_replicas_states` raises an exception if it fails, so the handling is likely incorrect here.




Discussion:

Resolving patch:
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -303,9 +303,13 @@ def _pick_random_rse(rse_expression):
                 if not no_register:
                     if register_after_upload:
                         self._register_file(file, registered_dataset_dids)
-                    replica_for_api = self._convert_file_for_api(file)
-                    if not self.client.update_replicas_states(rse, files=[replica_for_api]):
-                        logger(logging.WARNING, 'Failed to update replica state')
+                    else:
+                        replica_for_api = self._convert_file_for_api(file)
+                        try:
+                            self.client.update_replicas_states(rse, files=[replica_for_api])
+                        except Exception as error:
+                            logger(logging.ERROR, 'Failed to update replica state for file {}'.format(basename))
+                            logger(logging.DEBUG, 'Details: {}'.format(str(error)))
 
                 # add file to dataset if needed
                 if dataset_did_str and not no_register:


Test patch:
Repository: rucio/rucio

Issue:
ATLAS: Do not restrict viewing the global account usage
Motivation
----------

The local account usage is viewable from all users:

https://github.com/rucio/rucio/blob/b6092bcd78f8cfac079e5b3cc25a92b4163960ed/lib/rucio/core/permission/atlas.py#L1050-L1058

Yet, the global account usage is restricted to accounts with elevated privileges:

https://github.com/rucio/rucio/blob/b6092bcd78f8cfac079e5b3cc25a92b4163960ed/lib/rucio/core/permission/generic.py#L880-L895

There does not appear to be a good reason about this.

Modification
------------

Make the appropriate changes to the permissions.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/permission/atlas.py b/lib/rucio/core/permission/atlas.py
--- a/lib/rucio/core/permission/atlas.py
+++ b/lib/rucio/core/permission/atlas.py
@@ -26,16 +26,20 @@
 #
 # PY3K COMPATIBLE
 
+from typing import TYPE_CHECKING
+
 import rucio.core.did
 import rucio.core.scope
 from rucio.core.account import list_account_attributes, has_account_attribute
 from rucio.core.identity import exist_identity_account
-from rucio.core.permission.generic import perm_get_global_account_usage
 from rucio.core.rse import list_rse_attributes
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.rule import get_rule
 from rucio.db.sqla.constants import IdentityType, BadPFNStatus
 
+if TYPE_CHECKING:
+    from typing import Dict
+
 
 def has_permission(issuer, action, kwargs):
     """
@@ -1058,6 +1062,17 @@ def perm_get_local_account_usage(issuer, kwargs):
     return True
 
 
+def perm_get_global_account_usage(issuer: str, kwargs: 'Dict[str, str]') -> bool:
+    """
+    Checks if an account can get the account usage of an account.
+
+    :param issuer: Account identifier which issues the command.
+    :param kwargs: List of arguments for the action.
+    :returns: True if account is allowed, otherwise False
+    """
+    return True
+
+
 def perm_add_account_attribute(issuer, kwargs):
     """
     Checks if an account can add attributes to accounts.


Test patch:
Repository: rucio/rucio

Issue:
Bug affecting detach in delete_did
Motivation
----------
Undertaker : Bug affecting detach in delete_did
```
Undertaker(0): Got database error Database exception.
Details: (cx_Oracle.NotSupportedError) Python value of type DIDType not supported.
```





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -697,7 +697,7 @@ def delete_dids(dids, account, expire_rules=False, session=None, logger=logging.
         with record_timer_block('undertaker.parent_content'):
             for parent_did in session.query(models.DataIdentifierAssociation).filter(or_(*parent_content_clause)):
                 existing_parent_dids = True
-                detach_dids(scope=parent_did.scope, name=parent_did.name, dids=[{'scope': parent_did.child_scope, 'name': parent_did.child_type}], session=session)
+                detach_dids(scope=parent_did.scope, name=parent_did.name, dids=[{'scope': parent_did.child_scope, 'name': parent_did.child_name}], session=session)
 
     # Set Epoch tombstone for the files replicas inside the did
     if config_core.get('undertaker', 'purge_all_replicas', default=False, session=session) and file_content_clause:


Test patch:
Repository: rucio/rucio

Issue:
Improve AD32 checksum calculation
Motivation
----------
Right now the function relies on newlines. This is error-prone and can easily cause problems with binary files.


Modification
------------

Memory map the file and do binary block calculation.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -47,8 +47,10 @@
 import errno
 import getpass
 import hashlib
+import io
 import json
 import logging
+import mmap
 import os
 import os.path
 import re
@@ -59,6 +61,7 @@
 import time
 import zlib
 from enum import Enum
+from functools import partial
 from uuid import uuid4 as uuid
 from xml.etree import ElementTree
 
@@ -239,13 +242,16 @@ def adler32(file):
     adler = 1
 
     try:
-        with open(file, 'rb') as openFile:
-            for line in openFile:
-                adler = zlib.adler32(line, adler)
+        with open(file, 'r+b') as f:
+            # memory map the file
+            m = mmap.mmap(f.fileno(), 0)
+            # partial block reads at slightly increased buffer sizes
+            for block in iter(partial(m.read, io.DEFAULT_BUFFER_SIZE), b''):
+                adler = zlib.adler32(block, adler)
     except Exception as e:
         raise Exception('FATAL - could not get Adler32 checksum of file %s - %s' % (file, e))
 
-    # backflip on 32bit
+    # backflip on 32bit -- can be removed once everything is fully migrated to 64bit
     if adler < 0:
         adler = adler + 2 ** 32
 


Test patch:
Repository: rucio/rucio

Issue:
Introduce configurable timeout for deletion
Motivation
----------
The current timeout for the deletion in gfal is too long (8000 seconds for HTTP). Make it configurable







Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/protocols/gfal.py b/lib/rucio/rse/protocols/gfal.py
--- a/lib/rucio/rse/protocols/gfal.py
+++ b/lib/rucio/rse/protocols/gfal.py
@@ -63,6 +63,8 @@
         if os.environ['RUCIO_CLIENT_MODE']:
             raise exception.MissingDependency('Missing dependency : gfal2')
 
+TIMEOUT = config.config_get('deletion', 'timeout', False, None)
+
 
 class Default(protocol.RSEProtocol):
     """ Implementing access to RSEs using the srm protocol."""
@@ -207,6 +209,14 @@ def connect(self):
         self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False)
         if self.auth_token:
             self.__ctx.set_opt_string("BEARER", "TOKEN", self.auth_token)
+        if TIMEOUT:
+            try:
+                timeout = int(TIMEOUT)
+                self.__ctx.set_opt_integer("HTTP PLUGIN", "OPERATION_TIMEOUT", timeout)
+                self.__ctx.set_opt_integer("SRM PLUGIN", "OPERATION_TIMEOUT", timeout)
+                self.__ctx.set_opt_integer("GSIFTP PLUGIN", "OPERATION_TIMEOUT", timeout)
+            except ValueError:
+                self.logger(logging.ERROR, 'wrong timeout value %s', TIMEOUT)
 
     def get(self, path, dest, transfer_timeout=None):
         """


Test patch:
Repository: rucio/rucio

Issue:
Pause evaluating DIDs in case of Unique Constraint hit
Motivation
----------
The judge-evaluator is pausing the evaluation of dids (for a certain) time if certain error conditions are hit.
https://github.com/rucio/rucio/blob/2555f0a967bfa52c7486441cc704609800726151/lib/rucio/daemons/judge/evaluator.py#L129

Modification
------------
Also add the condition `cx_Oracle.IntegrityError: ORA-00001: unique constraint (ATLAS_RUCIO.REPLICAS_PK) violated` to this.



Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/judge/evaluator.py b/lib/rucio/daemons/judge/evaluator.py
--- a/lib/rucio/daemons/judge/evaluator.py
+++ b/lib/rucio/daemons/judge/evaluator.py
@@ -25,6 +25,7 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 """
 Judge-Evaluator is a daemon to re-evaluate and execute replication rules.
@@ -126,7 +127,7 @@ def re_evaluator(once=False, sleep_time=30, did_limit=100):
                     except DataIdentifierNotFound:
                         delete_updated_did(id_=did.id)
                     except (DatabaseException, DatabaseError) as e:
-                        if match('.*ORA-00054.*', str(e.args[0])):
+                        if match('.*ORA-000(01|54).*', str(e.args[0])):
                             paused_dids[(did.scope.internal, did.name)] = datetime.utcnow() + timedelta(seconds=randint(60, 600))
                             logging.warning('re_evaluator[%s/%s]: Locks detected for %s:%s' % (heartbeat['assign_thread'], heartbeat['nr_threads'], did.scope, did.name))
                             record_counter('rule.judge.exceptions.{exception}', labels={'exception': 'LocksDetected'})


Test patch: