internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: rucio/rucio

Issue:
Possibility to specify push or pull mode for WebDAV transfers
Motivation
----------
Provide the possibility to specify push or pull mode for WebDAV transfers in conveyor.







Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -91,6 +91,7 @@
                                        expiration_time=600,
                                        arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True})
 TRANSFER_TOOL = config_get('conveyor', 'transfertool', False, None)
+WEBDAV_TRANSFER_MODE = config_get('conveyor', 'webdav_transfer_mode', False, None)
 
 
 def submit_bulk_transfers(external_host, files, transfertool='fts3', job_params={}, timeout=None, user_transfer_job=False):
@@ -776,6 +777,11 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                 if sign_url == 'gcs':
                     dest_url = re.sub('davs', 'gclouds', dest_url)
                     dest_url = re.sub('https', 'gclouds', dest_url)
+                    if source_protocol in ['davs', 'https']:
+                        source_url += '?copy_mode=push'
+                elif WEBDAV_TRANSFER_MODE:
+                    if source_protocol in ['davs', 'https']:
+                        source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
 
                 # IV - get external_host + strict_copy
                 strict_copy = rse_attrs[dest_rse_id].get('strict_copy', False)
@@ -911,6 +917,15 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
 
                 # II - Build the source URL
                 source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0]
+                sign_url = rse_attrs[dest_rse_id].get('sign_url', None)
+                if sign_url == 'gcs':
+                    dest_url = re.sub('davs', 'gclouds', dest_url)
+                    dest_url = re.sub('https', 'gclouds', dest_url)
+                    if source_protocol in ['davs', 'https']:
+                        source_url += '?copy_mode=push'
+                elif WEBDAV_TRANSFER_MODE:
+                    if source_protocol in ['davs', 'https']:
+                        source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
 
                 # III - The transfer queued previously is a multihop, but this one is direct.
                 # Reset the sources, remove the multihop flag


Test patch:
Repository: rucio/rucio

Issue:
Problems with get_hops
Motivation
----------
Debugging with Katy her attempt to multi hop, we noticed that while there is a check in non-multihopping code which ignores distances set to None, there is no check in the multi hop link finder. We're now running a version which ignores such links in its test.

I also noticed the function is using [] as a default argument, which is a python no-no.

I can make a PR that fixes both issues. Hopefully after we get MH working.


Modification
------------





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -455,7 +455,7 @@ def update_transfer_state(external_host, transfer_id, state, logging_prepend_str
 
 
 @transactional_session
-def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=[], session=None):
+def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=None, session=None):
     """
     Get a list of hops needed to transfer date from source_rse_id to dest_rse_id.
     Ideally, the list will only include one item (dest_rse_id) since no hops are needed.
@@ -468,6 +468,9 @@ def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=[
     :raises:                   NoDistance
     """
 
+    if multihop_rses is None:
+        multihop_rses = []
+
     # TODO: Might be problematic to always load the distance_graph, since it might be expensive
     # TODO: Have an rse_expression to specify the eligible hops
 
@@ -522,6 +525,8 @@ def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=[
 
             for out_v in distance_graph[current_node]:
                 # Check if the distance would be smaller
+                if distance_graph[current_node][out_v] is None:
+                    continue
                 if visited_nodes.get(out_v, {'distance': 9999})['distance'] > current_distance + distance_graph[current_node][out_v] + HOP_PENALTY\
                    and local_optimum > current_distance + distance_graph[current_node][out_v] + HOP_PENALTY:
                     # Check if the intermediate RSE is enabled for multihop
@@ -655,6 +660,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
         try:
             list_hops = get_hops(source_rse_id, dest_rse_id, include_multihop=core_config_get('transfers', 'use_multihop', default=False, expiration_time=600, session=session), multihop_rses=multihop_rses, session=session)
             if len(list_hops) > 1:
+                logging.debug('From %s to %s requires multihop: %s', source_rse_id, dest_rse_id, list_hops)
                 multihop = True
                 multi_hop_dict[req_id] = (list_hops, dict_attributes, retry_count)
         except NoDistance:


Test patch:
Repository: rucio/rucio

Issue:
Add hint on query to get the number of replicas
Motivation
----------
A hint on REPLICAS_PK needs to be added on the query to get the number of replicas in `list_and_mark_unlocked_replicas`





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1665,6 +1665,7 @@ def list_and_mark_unlocked_replicas(limit, bytes=None, rse_id=None, delay_second
     for (scope, name, path, bytes, tombstone, state) in query.yield_per(1000):
         # Check if more than one replica is available
         replica_cnt = session.query(func.count(models.RSEFileAssociation.scope)).\
+            with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
             filter(and_(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id != rse_id)).one()
 
         if replica_cnt[0] > 1:


Test patch:
Repository: rucio/rucio

Issue:
Better handling of posix replica URLs
Motivation
----------

With `rucio.rse.protocols.posix.Default` implementation, the replicas with "file" schema always have port `0` in them, eg

`TEST1: file://:0/tmp/testfile`

Modification
------------

Override `lfns2pfns()` in the posix protocol.

I will submit a pull request for this shortly.



Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/protocols/posix.py b/lib/rucio/rse/protocols/posix.py
--- a/lib/rucio/rse/protocols/posix.py
+++ b/lib/rucio/rse/protocols/posix.py
@@ -8,6 +8,7 @@
 # Authors:
 # - Ralph Vigne, <ralph.vigne@cern.ch>, 2012-2014
 # - Nicolo Magini, <nicolo.magini@cern.ch>, 2018
+# - David Cameron, <david.cameron@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -158,6 +159,43 @@ def rename(self, pfn, new_pfn):
             else:
                 raise exception.ServiceUnavailable(e)
 
+    def lfns2pfns(self, lfns):
+        """ Returns fully qualified PFNs for the file referred by each lfn in
+            the lfns list.
+
+            :param lfns: List of lfns. If lfn['path'] is present it is used as
+                   the path to the file, otherwise the path is constructed
+                   deterministically.
+
+            :returns: Fully qualified PFNs.
+        """
+        pfns = {}
+        prefix = self.attributes['prefix']
+
+        if not prefix.startswith('/'):
+            prefix = ''.join(['/', prefix])
+        if not prefix.endswith('/'):
+            prefix = ''.join([prefix, '/'])
+
+        lfns = [lfns] if isinstance(lfns, dict) else lfns
+        for lfn in lfns:
+            scope, name = str(lfn['scope']), lfn['name']
+            if lfn.get('path'):
+                pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'],
+                                                         '://',
+                                                         self.attributes['hostname'],
+                                                         prefix,
+                                                         lfn['path'] if not lfn['path'].startswith('/') else lfn['path'][1:]
+                                                         ])
+            else:
+                pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'],
+                                                         '://',
+                                                         self.attributes['hostname'],
+                                                         prefix,
+                                                         self._get_path(scope=scope, name=name)
+                                                         ])
+        return pfns
+
     def pfn2path(self, pfn):
         tmp = list(self.parse_pfns(pfn).values())[0]
         return '/'.join([tmp['prefix'], tmp['path'], tmp['name']])


Test patch:
Repository: rucio/rucio

Issue:
Fix Belle2 schema for RSE and RSE attributes
Motivation
----------
The current schema for RSE and RSE attributes break the creation of the rules. Needs to be fixed




Discussion:

Resolving patch:
diff --git a/lib/rucio/common/schema/belleii.py b/lib/rucio/common/schema/belleii.py
--- a/lib/rucio/common/schema/belleii.py
+++ b/lib/rucio/common/schema/belleii.py
@@ -62,7 +62,7 @@
 
 NAME = {"description": "Data Identifier name",
         "type": "string",
-        "pattern": r"^[\/A-Za-z0-9][A-Za-z0-9\\.\\-\\_\/]{1,%s}$" % NAME_LENGTH}
+        "pattern": r"^/[A-Za-z0-9\.\-\_\+\/]{1,%s}$" % NAME_LENGTH}
 
 R_NAME = {"description": "Data Identifier name",
           "type": "string",
@@ -93,7 +93,7 @@
 
 DEFAULT_RSE_ATTRIBUTE = {"description": "Default RSE attribute",
                          "type": "string",
-                         "pattern": r'([A-Z0-9]+([_-][A-Z0-9]+)*)'}
+                         "pattern": r'([A-Za-z0-9]+([_-][A-Za-z0-9]+)*)'}
 
 REPLICA_STATE = {"description": "Replica state",
                  "type": "string",
@@ -271,7 +271,7 @@
                              "name": NAME,
                              "rse": {"description": "RSE name",
                                      "type": ["string", "null"],
-                                     "pattern": "^([A-Z0-9]+([_-][A-Z0-9]+)*)$"},
+                                     "pattern": "^([A-Z])+((([A-Za-z0-9]+))*(-)?)*$"},
                              "dids": DIDS},
               "required": ["dids"],
               "additionalProperties": False}


Test patch:
Repository: rucio/rucio

Issue:
grouping value is corrupted in case of add_rule call with multiple kids
Motivation
----------
it will always become DATASET from second did on


Modification
------------
move grouping translation line out of loop




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -24,6 +24,7 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Luc Goossens <luc.goossens@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -136,6 +137,8 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
     """
     rule_ids = []
 
+    grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(grouping, RuleGrouping.DATASET)
+
     with record_timer_block('rule.add_rule'):
         # 1. Resolve the rse_expression into a list of RSE-ids
         with record_timer_block('rule.add_rule.parse_rse_expression'):
@@ -221,8 +224,6 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
 
             # 4. Create the replication rule
             with record_timer_block('rule.add_rule.create_rule'):
-                grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(grouping, RuleGrouping.DATASET)
-
                 if meta is not None:
                     try:
                         meta = json.dumps(meta)


Test patch:
Repository: rucio/rucio

Issue:
get_did returns None for bytes instead of zero if dataset has no files yet
Motivation
----------
this requires additional testing on clients to avoid numeric exceptions


Modification
------------





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -25,8 +25,9 @@
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Tobias Wegner <twegner@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
-# - Ruturaj Gujar, <ruturaj.gujar23@gmail.com>, 2019
-# - Brandon White, <bjwhite@fnal.gov>, 2019
+# - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Goossens Luc <luc.goossens@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -1142,6 +1143,9 @@ def get_did(scope, name, dynamic=False, session=None):
         else:
             if dynamic:
                 bytes, length, events = __resolve_bytes_length_events_did(scope=scope, name=name, session=session)
+                # replace None value for bytes with zero
+                if bytes is None:
+                    bytes = 0
             else:
                 bytes, length = result.bytes, result.length
             return {'scope': result.scope, 'name': result.name, 'type': result.did_type,


Test patch:
Repository: rucio/rucio

Issue:
list_child_datasets does not return a set
Motivation
----------
if the container graph is not a tree, the result contains duplicate entries for datasets contained in multiple child containers


Modification
------------
filter out duplicates before returning result or never add them in the first place




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -25,8 +25,9 @@
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Tobias Wegner <twegner@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
-# - Ruturaj Gujar, <ruturaj.gujar23@gmail.com>, 2019
-# - Brandon White, <bjwhite@fnal.gov>, 2019
+# - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Luc Goossens <luc.goossens@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -974,6 +975,10 @@ def list_child_datasets(scope, name, session=None):
             result.extend(list_child_datasets(scope=child_scope, name=child_name, session=session))
         else:
             result.append({'scope': child_scope, 'name': child_name, 'type': child_type})
+
+    # remove duplicate entries
+    result = {(elem['scope'], elem['name']): elem for elem in result}.values()
+
     return result
 
 


Test patch:
Repository: rucio/rucio

Issue:
setup.py for main rucio package is not Python 3 compatible
Motivation
----------
The `setup.py` for the main Rucio package is not Python 3 compatible. 

Modification
------------
- [x] Fix compatibility
- [x] Add Python3 Test for server to travis



Discussion:

Resolving patch:
diff --git a/setup_rucio.py b/setup_rucio.py
--- a/setup_rucio.py
+++ b/setup_rucio.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2016-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2016-2020
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Ruturaj Gujar, <ruturaj.gujar23@gmail.com>, 2019
@@ -102,7 +102,7 @@ def run_git_command(cmd):
                        """}""" % (pkg_version, branch_nick, revid, revno))
     version_file.close()
 else:
-    pkg_version = version.version_string().decode()
+    pkg_version = version.version_string()
 
 cmdclass = {}
 
diff --git a/tools/generate_version.py b/tools/generate_version.py
--- a/tools/generate_version.py
+++ b/tools/generate_version.py
@@ -22,13 +22,13 @@ def run_git_command(cmd):
         GIT_VERSION = sys.argv[1]
     else:
         GIT_VERSION_CMD = 'git describe --abbrev=4'
-        GIT_VERSION = run_git_command(GIT_VERSION_CMD)
+        GIT_VERSION = run_git_command(GIT_VERSION_CMD).decode()
     BRANCH_NICK_CMD = 'git branch | grep -Ei "\* (.*)" | cut -f2 -d" "'  # NOQA: W605
-    BRANCH_NICK = run_git_command(BRANCH_NICK_CMD)
+    BRANCH_NICK = run_git_command(BRANCH_NICK_CMD).decode()
     REVID_CMD = "git rev-parse HEAD"
-    REVID = run_git_command(REVID_CMD)
+    REVID = run_git_command(REVID_CMD).decode()
     REVNO_CMD = "git --no-pager log --oneline | wc -l"
-    REVNO = run_git_command(REVNO_CMD)
+    REVNO = run_git_command(REVNO_CMD).decode()
     VERSION_FILE = open("lib/rucio/vcsversion.py", 'w')
     VERSION_FILE.write("""
 '''


Test patch:
Repository: rucio/rucio

Issue:
Rucio client package imports rucio.common.schema which is not part of distribution
Motivation
----------
From Isabel:
```
cmd: rucio upload --scope user.itrigger --rse TRIUMF-LCG2_SCRATCHDISK /tmp/itrigger/itrigger/.alrb/testing/tester.NKHUnK/rucio/bash/rucioUpload
2020-04-06 18:30:38,129
INFO Preparing upload for file myFile.d314ec0d-6684-4e3f-8501-47467586e335.log
2020-04-06 18:30:38,293
INFO Successfully added replica in Rucio catalogue at TRIUMF-LCG2_SCRATCHDISK
2020-04-06 18:30:38,842
INFO Successfully added replication rule at TRIUMF-LCG2_SCRATCHDISK
2020-04-06 18:30:39,962
ERROR No module named schema
2020-04-06 18:30:39,963
ERROR 
Rucio exited with an unexpected/unknown error.
Please rerun the last command with the "-v" option to gather more information.
If you're sure there is a problem with Rucio itself, please followup at: https://github.com/rucio/rucio/issues
rucio upload dataset (bash)                                         [FAILED]
```
For some reason the schema module seems to be imported, which is not part of the client distribution.

Modification
------------
We can add it to the distribution, but for me it is not clear why this is even needed, so we should understand this first.



Discussion:
line where we import the schema in client mode:
https://github.com/rucio/rucio/blob/next/lib/rucio/rse/protocols/protocol.py#L230
Ok, I followed this up and I think in this case we have to include common.schema as well to the clients package, as it is needed for the VO policy packages. I will prepare a PR.

Resolving patch:
diff --git a/setup_rucio_client.py b/setup_rucio_client.py
--- a/setup_rucio_client.py
+++ b/setup_rucio_client.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2017-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2017-2020
 
 import os
 import re
@@ -39,7 +39,7 @@
 COPY_ARGS = sys.argv[1:]
 NAME = 'rucio-clients'
 IS_RELEASE = False
-PACKAGES = ['rucio', 'rucio.client', 'rucio.common',
+PACKAGES = ['rucio', 'rucio.client', 'rucio.common', 'rucio.common.schema', 
             'rucio.rse.protocols', 'rucio.rse', 'rucio.tests']
 REQUIREMENTS_FILES = ['etc/pip-requires-client']
 DESCRIPTION = "Rucio Client Lite Package"


Test patch:
Repository: rucio/rucio

Issue:
Fix source scheme in case of transfers from GCS
Motivation
----------
Fix source scheme in case of transfers from GCS





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -800,9 +800,16 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                     if source_protocol in ['davs', 'https']:
                         source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
 
+                source_sign_url = rse_attrs[source_rse_id].get('sign_url', None)
+                if source_sign_url == 'gcs':
+                    source_url = re.sub('davs', 'gclouds', source_url)
+                    source_url = re.sub('https', 'gclouds', source_url)
+
                 # IV - get external_host + strict_copy
                 strict_copy = rse_attrs[dest_rse_id].get('strict_copy', False)
                 fts_hosts = rse_attrs[dest_rse_id].get('fts', None)
+                if source_sign_url == 'gcs':
+                    fts_hosts = rse_attrs[source_rse_id].get('fts', None)
                 source_globus_endpoint_id = rse_attrs[source_rse_id].get('globus_endpoint_id', None)
                 dest_globus_endpoint_id = rse_attrs[dest_rse_id].get('globus_endpoint_id', None)
 
@@ -946,6 +953,11 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                     if source_protocol in ['davs', 'https']:
                         source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
 
+                source_sign_url = rse_attrs[source_rse_id].get('sign_url', None)
+                if source_sign_url == 'gcs':
+                    source_url = re.sub('davs', 'gclouds', source_url)
+                    source_url = re.sub('https', 'gclouds', source_url)
+
                 # III - The transfer queued previously is a multihop, but this one is direct.
                 # Reset the sources, remove the multihop flag
                 if transfers[req_id].get('multihop', False):


Test patch:
Repository: rucio/rucio

Issue:
Add "Globus Online Test" activity to schema (ATLAS)
Motivation
----------
Add `Globos Online Test` activity to ATLAS schema




Discussion:

Resolving patch:
diff --git a/lib/rucio/common/schema/atlas.py b/lib/rucio/common/schema/atlas.py
--- a/lib/rucio/common/schema/atlas.py
+++ b/lib/rucio/common/schema/atlas.py
@@ -46,7 +46,8 @@
                      "Production Input", "Production Output",
                      "Analysis Input", "Analysis Output", "Staging",
                      "T0 Export", "T0 Tape", "Upload/Download (Job)",
-                     "Upload/Download (User)", "User Subscriptions"]}
+                     "Upload/Download (User)", "User Subscriptions",
+                     "Globus Online Test"]}
 
 SCOPE_LENGTH = 25
 


Test patch:
Repository: rucio/rucio

Issue:
Add "Globus Online Test" activity to schema (ATLAS)
Motivation
----------
Add `Globos Online Test` activity to ATLAS schema




Discussion:

Resolving patch:
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.22.2',
-    'branch_nick': 'patch-0-Release__1_22_2_preparation',
-    'revision_id': '1640edd8514f74d444775b5d80ec864764edef0a',
-    'revno': 9385
+    'version': '1.22.3',
+    'branch_nick': 'patch-0-Release_1_22_3_preparation',
+    'revision_id': '73c443db84fde6b03b489595c42e7a39221f55e1',
+    'revno': 9419
 }


Test patch: