Repository: rucio/rucio

Issue:
create_did_sample broken for DIDs with "/"
Motivation
----------
`create_did_sample` is broken for DIDs with "/" (problem at the REST layer), so cannot be used for policies with `SCOPE_NAME_REGEXP = '/([^/]*)(?=/)(.*)'` (Belle II, CMS, Icecube, etc.)




Discussion:
https://github.com/rucio/rucio/blob/937c75344c68a143c278e42fa853918eb452b56a/lib/rucio/web/ui/static/rucio.js#L1387-L1405

the webui uses `create_did_sample` too, this needs to be reworked (and tested!!)

Resolving patch:
diff --git a/lib/rucio/client/didclient.py b/lib/rucio/client/didclient.py
--- a/lib/rucio/client/didclient.py
+++ b/lib/rucio/client/didclient.py
@@ -663,9 +663,16 @@ def create_did_sample(self, input_scope, input_name, output_scope, output_name,
         :param account: The account.
         :param nbfiles: The number of files to register in the output dataset.
         """
-        path = '/'.join([self.DIDS_BASEURL, quote_plus(input_scope), quote_plus(input_name), quote_plus(output_scope), quote_plus(output_name), str(nbfiles), 'sample'])
+        path = '/'.join([self.DIDS_BASEURL, 'sample'])
+        data = dumps({
+            'input_scope': input_scope,
+            'input_name': input_name,
+            'output_scope': output_scope,
+            'output_name': output_name,
+            'nbfiles': str(nbfiles)
+        })
         url = build_url(choice(self.list_hosts), path=path)
-        r = self._send_request(url, type_='POST', data=dumps({}))
+        r = self._send_request(url, type_='POST', data=data)
         if r.status_code == codes.created:
             return True
         else:
diff --git a/lib/rucio/web/rest/flaskapi/v1/dids.py b/lib/rucio/web/rest/flaskapi/v1/dids.py
--- a/lib/rucio/web/rest/flaskapi/v1/dids.py
+++ b/lib/rucio/web/rest/flaskapi/v1/dids.py
@@ -1848,7 +1848,7 @@ def generate(vo):
             return generate_http_error_flask(404, error)
 
 
-class Sample(ErrorHandlingMethodView):
+class SampleLegacy(ErrorHandlingMethodView):
 
     def post(self, input_scope, input_name, output_scope, output_name, nbfiles):
         """
@@ -1925,6 +1925,81 @@ def post(self, input_scope, input_name, output_scope, output_name, nbfiles):
         return 'Created', 201
 
 
+class Sample(ErrorHandlingMethodView):
+
+    def post(self):
+        """
+        ---
+        summary: Create sample
+        description: Creates a sample from an input collection.
+        tags:
+          - Data Identifiers
+        requestBody:
+          description: Parameters (source and destination) for the files in the sample to be created
+          content:
+            'application/json':
+              schema:
+                type: object
+                required:
+                - input_scope
+                - input_name
+                - output_scope
+                - output_name
+                - nbfiles
+                properties:
+                  input_scope:
+                    description: The input scope.
+                    type: string
+                  input_name:
+                    description: The input name.
+                    type: string
+                  output_scope:
+                    description: The output scope.
+                    type: string
+                  output_name:
+                    description: The output name.
+                    type: string
+                  nbfiles:
+                    description: The number of files to register in the output dataset.
+                    type: string
+        responses:
+          201:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ["Created"]
+          401:
+            description: Invalid Auth Token
+          404:
+            description: Not found
+          406:
+            description: Not acceptable
+          409:
+            description: Duplication
+        """
+        parameters = json_parameters()
+        try:
+            create_did_sample(
+                input_scope=parameters['input_scope'],
+                input_name=parameters['input_name'],
+                output_scope=parameters['output_scope'],
+                output_name=parameters['output_name'],
+                issuer=request.environ.get('issuer'),
+                nbfiles=parameters['nbfiles'],
+                vo=request.environ.get('vo'),
+            )
+        except DataIdentifierNotFound as error:
+            return generate_http_error_flask(404, error)
+        except (DuplicateContent, DataIdentifierAlreadyExists, UnsupportedOperation) as error:
+            return generate_http_error_flask(409, error)
+        except AccessDenied as error:
+            return generate_http_error_flask(401, error)
+
+        return 'Created', 201
+
+
 class NewDIDs(ErrorHandlingMethodView):
 
     @check_accept_header_wrapper_flask(['application/x-json-stream'])
@@ -2225,8 +2300,10 @@ def blueprint():
     bp.add_url_rule('/<path:scope_name>', view_func=dids_view, methods=['get', 'post'])
     bulkdids_view = BulkDIDS.as_view('bulkdids')
     bp.add_url_rule('', view_func=bulkdids_view, methods=['post', ])
-    sample_view = Sample.as_view('sample')
-    bp.add_url_rule('/<input_scope>/<input_name>/<output_scope>/<output_name>/<nbfiles>/sample', view_func=sample_view, methods=['post', ])
+    sample_view_legacy = SampleLegacy.as_view('sample')
+    bp.add_url_rule('/<input_scope>/<input_name>/<output_scope>/<output_name>/<nbfiles>/sample', view_func=sample_view_legacy, methods=['post', ])
+    sample_view = Sample.as_view('sample_new')
+    bp.add_url_rule('/sample', view_func=sample_view, methods=['post', ])
     attachements_view = Attachments.as_view('attachments')
     bp.add_url_rule('/attachments', view_func=attachements_view, methods=['post', ])
     new_dids_view = NewDIDs.as_view('new_dids')


Test patch:
diff --git a/lib/rucio/tests/test_did.py b/lib/rucio/tests/test_did.py
--- a/lib/rucio/tests/test_did.py
+++ b/lib/rucio/tests/test_did.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 
 from datetime import datetime, timedelta
-import os
 
 import pytest
 
@@ -529,7 +528,6 @@ def test_add_did(self, vo, did_client, rse_factory):
         did_client.close(scope=tmp_scope, name=tmp_dsn)
 
     @pytest.mark.dirty
-    @pytest.mark.skipif(os.environ.get('POLICY') != 'atlas', reason='REST API only works with ATLAS DID convention')
     @pytest.mark.noparallel(reason='uses pre-defined scope')
     def test_create_sample(self, vo, root_account, did_client, rse_factory):
         """ DATA IDENTIFIERS (CLIENT): Create a sample"""
diff --git a/tools/test/prepare_db_for_ui.sh b/tools/test/prepare_db_for_ui.sh
new file mode 100755
--- /dev/null
+++ b/tools/test/prepare_db_for_ui.sh
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ACCOUNT=anton
+IDENTITY=antonid
+EMAIL='antonmail@mail.ch'
+PASSWORD=anton
+SCOPE=user.anton
+DATASET=antondset
+RSE=XRD1
+
+rucio-admin account add $ACCOUNT
+rucio-admin account add-attribute --key 'admin' --value true $ACCOUNT
+
+# add userpass identity
+rucio-admin identity add --account $ACCOUNT --type USERPASS --id $IDENTITY --email $EMAIL --pass $PASSWORD
+
+# ( logging into rucio UI requires you to use the IDENTITY not the ACCOUNT)
+
+# create scope antonscope
+rucio-admin scope add --account $ACCOUNT --scope $SCOPE
+rucio-admin scope add --account $ACCOUNT --scope user.$ACCOUNT
+# test using rucio list-scopes
+
+# add dataset antondset
+rucio add-dataset $SCOPE:$DATASET
+
+# create test files numbered tfile1.txt through tfile3.txt
+echo 'Creating test files'
+echo {1..3} | sed -e 's/\s/\n/g' | xargs -I{} sh -c "echo hello test no {} >> tfile{}.txt"
+
+rucio upload --scope $SCOPE --rse $RSE tfile*
+
+rucio-admin account set-limits $ACCOUNT $RSE infinity
+
+rucio attach $SCOPE:$DATASET $SCOPE:tfile{1..3}.txt
+
+rucio add-rule $SCOPE:$DATASET 1 $RSE