Repository: rucio/rucio

Issue:
Wrong cms RSE name pattern used when attaching dids
Motivation
----------
in lib/rucio/common/schema/cms.py the RSE Name pattern is 

`^T[0-3]_[A-Z]{2}((_[A-Za-z0-9]+)+)$`

while the pattern for the rse property of ATTACHMENT is

`^([A-Z0-9]+([_-][A-Z0-9]+)*)$`

as a result when we add replicas to a CMS RSE whose name is not all in capitals (e.g. T1_UK_RAL_Buffer):
  * if we use `add_replicas` and the `attach_dids` it works fine
  * if we use `attach_dids` directly it fails with `InvalidObject`

Modification
------------
Just put the same regexp for the two patterns (PR coming)




Discussion:
Created PR #1902 
Maybe we may want to test it on the CERN server before merging.
Hi Andrea,
You created this PR against the next branch (Thus it is treated as a feature). This will then only go in the Rucio 1.19.0 release in February 2019. Is this fine? Otherwise please re-create as a patch.
See https://github.com/rucio/rucio/blob/master/CONTRIBUTING.rst
Hi Martin, 
ok sorry. I'm recreating the PR.
Created #1904
That's against the **hotfix** branch 😄
I'll create the two PRs for **master** and **next**. 
Although I just saw you based your development off the **hotfix** branch (instead of master). It's not super clean to merge it like that. It would probably work fine, but we should stick to the normal workflow here.
I'm really sorry.. I've read the doc too quickly and understood patches go in hotfix (I do not know how, since is clearly stated). I'll recreate 2 PR one for next and one for master (both from a branch based on master, if I understand correctly)
Hi Andrea, no worries. The workflow we have is sadly not very simple for somebody new, since it is somewhat different than most projects do it, so I can understand the confusion. Just ping me on Slack if you have any question about the setup. 😄 

Resolving patch:
diff --git a/lib/rucio/common/schema/cms.py b/lib/rucio/common/schema/cms.py
--- a/lib/rucio/common/schema/cms.py
+++ b/lib/rucio/common/schema/cms.py
@@ -269,7 +269,7 @@
                              "name": NAME,
                              "rse": {"description": "RSE name",
                                      "type": ["string", "null"],
-                                     "pattern": "^([A-Z0-9]+([_-][A-Z0-9]+)*)$"},
+                                     "pattern": "^T[0-3]_[A-Z]{2}((_[A-Za-z0-9]+)+)$"},
                              "dids": DIDS},
               "required": ["dids"],
               "additionalProperties": False}
diff --git a/lib/rucio/common/schema/domatpc.py b/lib/rucio/common/schema/domatpc.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/common/schema/domatpc.py
@@ -0,0 +1,378 @@
+# Copyright 2017-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2019
+#
+# PY3K COMPATIBLE
+
+from jsonschema import validate, ValidationError
+
+from rucio.common.exception import InvalidObject
+
+
+ACCOUNT = {"description": "Account name",
+           "type": "string",
+           "pattern": "^[a-z0-9-_]{1,30}$"}
+
+ACCOUNTS = {"description": "Array of accounts",
+            "type": "array",
+            "items": ACCOUNT,
+            "minItems": 0,
+            "maxItems": 1000}
+
+
+ACCOUNT_TYPE = {"description": "Account type",
+                "type": "string",
+                "enum": ["USER", "GROUP", "SERVICE"]}
+
+ACTIVITY = {"description": "Activity name",
+            "type": "string",
+            "enum": ["Functional Test XrootD", "Functional Test WebDAV", "Stress Test"]}
+
+SCOPE_LENGTH = 25
+
+SCOPE = {"description": "Scope name",
+         "type": "string",
+         "pattern": "^[a-zA-Z_\\-.0-9]{1,%s}$" % SCOPE_LENGTH}
+
+R_SCOPE = {"description": "Scope name",
+           "type": "string",
+           "pattern": "\\w"}
+
+NAME_LENGTH = 250
+
+NAME = {"description": "Data Identifier name",
+        "type": "string",
+        "pattern": "^[A-Za-z0-9][A-Za-z0-9\\.\\-\\_]{1,%s}$" % NAME_LENGTH}
+
+R_NAME = {"description": "Data Identifier name",
+          "type": "string",
+          "pattern": "\\w"}
+
+LOCKED = {"description": "Rule locked status",
+          "type": ["boolean", "null"]}
+
+ASK_APPROVAL = {"description": "Rule approval request",
+                "type": ["boolean", "null"]}
+
+ASYNCHRONOUS = {"description": "Asynchronous rule creation",
+                "type": ["boolean", "null"]}
+
+PURGE_REPLICAS = {"description": "Rule purge replica status",
+                  "type": "boolean"}
+
+IGNORE_AVAILABILITY = {"description": "Rule ignore availability status",
+                       "type": "boolean"}
+
+RSE = {"description": "RSE name",
+       "type": "string",
+       "pattern": "^([A-Z0-9]+([_-][A-Z0-9]+)*)$"}
+
+RSE_ATTRIBUTE = {"description": "RSE attribute",
+                 "type": "string",
+                 "pattern": r'([A-Za-z0-9\._-]+[=<>][A-Za-z0-9_-]+)'}
+
+DEFAULT_RSE_ATTRIBUTE = {"description": "Default RSE attribute",
+                         "type": "string",
+                         "pattern": r'([A-Z0-9]+([_-][A-Z0-9]+)*)'}
+
+REPLICA_STATE = {"description": "Replica state",
+                 "type": "string",
+                 "enum": ["AVAILABLE", "UNAVAILABLE", "COPYING", "BEING_DELETED", "BAD", "SOURCE", "A", "U", "C", "B", "D", "S"]}
+
+DATE = {"description": "Date",
+        "type": "string",
+        "pattern": r'((Mon)|(Tue)|(Wed)|(Thu)|(Fri)|(Sat)|(Sun))[,]\s\d{2}\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s(0\d|1\d|2[0-3])(\:)(0\d|1\d|2\d|3\d|4\d|5\d)(\:)(0\d|1\d|2\d|3\d|4\d|5\d)\s(UTC)'}
+
+DID_TYPE = {"description": "DID type",
+            "type": "string",
+            "enum": ["DATASET", "CONTAINER", "FILE", "F"]}
+
+GROUPING = {"description": "Rule grouping",
+            "type": ["string", "null"],
+            "enum": ["DATASET", "NONE", "ALL", None]}
+
+NOTIFY = {"description": "Rule notification setting",
+          "type": ["string", "null"],
+          "enum": ["Y", "C", "N", None]}
+
+COMMENT = {"description": "Rule comment",
+           "type": ["string", "null"],
+           "maxLength": 250}
+
+METADATA = {"description": "Rule wfms metadata",
+            "type": ["string", "null"],
+            "maxLength": 3999}
+
+BYTES = {"description": "Size in bytes",
+         "type": "integer"}
+
+ADLER32 = {"description": "adler32",
+           "type": "string",
+           "pattern": "^[a-fA-F\\d]{8}$"}
+
+WEIGHT = {"description": "Rule weight",
+          "type": ["string", "null"]}
+
+MD5 = {"description": "md5",
+       "type": "string",
+       "pattern": "^[a-fA-F\\d]{32}$"}
+
+UUID = {"description": "Universally Unique Identifier (UUID)",
+        "type": "string",
+        "pattern": '^(\\{){0,1}[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}(\\}){0,1}$'}
+
+META = {"description": "Data Identifier(DID) metadata",
+        "type": "object",
+        "properties": {"guid": UUID},
+        "additionalProperties": True}
+
+PFN = {"description": "Physical File Name", "type": "string"}
+
+COPIES = {"description": "Number of replica copies", "type": "integer"}
+
+RSE_EXPRESSION = {"description": "RSE expression", "type": "string"}
+
+SOURCE_REPLICA_EXPRESSION = {"description": "RSE expression", "type": ["string", "null"]}
+
+LIFETIME = {"description": "Lifetime", "type": "number"}
+
+RULE_LIFETIME = {"description": "Rule lifetime", "type": ["number", "null"]}
+
+SUBSCRIPTION_ID = {"description": "Rule Subscription id", "type": ["string", "null"]}
+
+PRIORITY = {"description": "Priority of the transfers",
+            "type": "integer"}
+
+SPLIT_CONTAINER = {"description": "Rule split container mode",
+                   "type": ["boolean", "null"]}
+
+RULE = {"description": "Replication rule",
+        "type": "object",
+        "properties": {"dids": {"type": "array"},
+                       "account": ACCOUNT,
+                       "copies": COPIES,
+                       "rse_expression": RSE_EXPRESSION,
+                       "grouping": GROUPING,
+                       "weight": WEIGHT,
+                       "lifetime": RULE_LIFETIME,
+                       "locked": LOCKED,
+                       "subscription_id": SUBSCRIPTION_ID,
+                       "source_replica_expression": SOURCE_REPLICA_EXPRESSION,
+                       "activity": ACTIVITY,
+                       "notify": NOTIFY,
+                       "purge_replicas": PURGE_REPLICAS,
+                       "ignore_availability": IGNORE_AVAILABILITY,
+                       "comment": COMMENT,
+                       "ask_approval": ASK_APPROVAL,
+                       "asynchronous": ASYNCHRONOUS,
+                       "priority": PRIORITY,
+                       'split_container': SPLIT_CONTAINER,
+                       'meta': METADATA},
+        "required": ["dids", "copies", "rse_expression"],
+        "additionalProperties": False}
+
+RULES = {"description": "Array of replication rules",
+         "type": "array",
+         "items": RULE,
+         "minItems": 1,
+         "maxItems": 1000}
+
+COLLECTION_TYPE = {"description": "Dataset or container type",
+                   "type": "string",
+                   "enum": ["DATASET", "CONTAINER"]}
+
+COLLECTION = {"description": "Dataset or container",
+              "type": "object",
+              "properties": {"scope": SCOPE,
+                             "name": NAME,
+                             "type": COLLECTION_TYPE,
+                             "meta": META,
+                             "rules": RULES},
+              "required": ["scope", "name", "type"],
+              "additionalProperties": False}
+
+COLLECTIONS = {"description": "Array of datasets or containers",
+               "type": "array",
+               "items": COLLECTION,
+               "minItems": 1,
+               "maxItems": 1000}
+
+DID = {"description": "Data Identifier(DID)",
+       "type": "object",
+       "properties": {"scope": SCOPE,
+                      "name": NAME,
+                      "type": DID_TYPE,
+                      "meta": META,
+                      "rules": RULES,
+                      "bytes": BYTES,
+                      "adler32": ADLER32,
+                      "md5": MD5,
+                      "state": REPLICA_STATE,
+                      "pfn": PFN},
+       "required": ["scope", "name"],
+       "additionalProperties": False}
+
+DID_FILTERS = {"description": "Filters dictionary to list DIDs",
+               "type": "object",
+               "properties": {"created_before": DATE,
+                              "created_afted": DATE},
+               "additionalProperties": True}
+
+R_DID = {"description": "Data Identifier(DID)",
+         "type": "object",
+         "properties": {"scope": R_SCOPE,
+                        "name": R_NAME,
+                        "type": DID_TYPE,
+                        "meta": META,
+                        "rules": RULES,
+                        "bytes": BYTES,
+                        "adler32": ADLER32,
+                        "md5": MD5,
+                        "state": REPLICA_STATE,
+                        "pfn": PFN},
+         "required": ["scope", "name"],
+         "additionalProperties": False}
+
+DIDS = {"description": "Array of Data Identifiers(DIDs)",
+        "type": "array",
+        "items": DID,
+        "minItems": 1,
+        "maxItems": 1000}
+
+R_DIDS = {"description": "Array of Data Identifiers(DIDs)",
+          "type": "array",
+          "items": R_DID,
+          "minItems": 1,
+          "maxItems": 1000}
+
+ATTACHMENT = {"description": "Attachement",
+              "type": "object",
+              "properties": {"scope": SCOPE,
+                             "name": NAME,
+                             "rse": {"description": "RSE name",
+                                     "type": ["string", "null"],
+                                     "pattern": "^([A-Z0-9]+([_-][A-Z0-9]+)*)$"},
+                             "dids": DIDS},
+              "required": ["dids"],
+              "additionalProperties": False}
+
+ATTACHMENTS = {"description": "Array of attachments",
+               "type": "array",
+               "items": ATTACHMENT,
+               "minItems": 1,
+               "maxItems": 1000}
+
+SUBSCRIPTION_FILTER = {"type": "object",
+                       "properties": {"datatype": {"type": "array"},
+                                      "prod_step": {"type": "array"},
+                                      "stream_name": {"type": "array"},
+                                      "project": {"type": "array"},
+                                      "scope": {"type": "array"},
+                                      "pattern": {"type": "string"},
+                                      "excluded_pattern": {"type": "string"},
+                                      "group": {"type": "string"},
+                                      "provenance": {"type": "string"},
+                                      "account": ACCOUNTS,
+                                      "grouping": {"type": "string"},
+                                      "split_rule": {"type": "boolean"}}}
+
+ADD_REPLICA_FILE = {"description": "add replica file",
+                    "type": "object",
+                    "properties": {"scope": SCOPE,
+                                   "name": NAME,
+                                   "bytes": BYTES,
+                                   "adler32": ADLER32},
+                    "required": ["scope", "name", "bytes", "adler32"]}
+
+ADD_REPLICA_FILES = {"description": "add replica files",
+                     "type": "array",
+                     "items": ADD_REPLICA_FILE,
+                     "minItems": 1,
+                     "maxItems": 1000}
+
+CACHE_ADD_REPLICAS = {"description": "rucio cache add replicas",
+                      "type": "object",
+                      "properties": {"files": ADD_REPLICA_FILES,
+                                     "rse": RSE,
+                                     "lifetime": LIFETIME,
+                                     "operation": {"enum": ["add_replicas"]}},
+                      "required": ['files', 'rse', 'lifetime', 'operation']}
+
+DELETE_REPLICA_FILE = {"description": "delete replica file",
+                       "type": "object",
+                       "properties": {"scope": SCOPE,
+                                      "name": NAME},
+                       "required": ["scope", "name"]}
+
+DELETE_REPLICA_FILES = {"description": "delete replica files",
+                        "type": "array",
+                        "items": DELETE_REPLICA_FILE,
+                        "minItems": 1,
+                        "maxItems": 1000}
+
+CACHE_DELETE_REPLICAS = {"description": "rucio cache delete replicas",
+                         "type": "object",
+                         "properties": {"files": DELETE_REPLICA_FILES,
+                                        "rse": RSE,
+                                        "operation": {"enum": ["delete_replicas"]}},
+                         "required": ['files', 'rse', 'operation']}
+
+MESSAGE_OPERATION = {"type": "object",
+                     "properties": {'operation': {"enum": ["add_replicas", "delete_replicas"]}}}
+
+ACCOUNT_ATTRIBUTE = {"description": "Account attribute",
+                     "type": "string",
+                     "pattern": r'^[a-z0-9-_]{1,30}$'}
+
+SCOPE_NAME_REGEXP = '/(.*)/(.*)'
+
+SCHEMAS = {'account': ACCOUNT,
+           'account_type': ACCOUNT_TYPE,
+           'activity': ACTIVITY,
+           'name': NAME,
+           'r_name': R_NAME,
+           'rse': RSE,
+           'rse_attribute': RSE_ATTRIBUTE,
+           'scope': SCOPE,
+           'r_scope': R_SCOPE,
+           'did': DID,
+           'did_filters': DID_FILTERS,
+           'r_did': R_DID,
+           'dids': DIDS,
+           'rule': RULE,
+           'r_dids': R_DIDS,
+           'collection': COLLECTION,
+           'collections': COLLECTIONS,
+           'attachment': ATTACHMENT,
+           'attachments': ATTACHMENTS,
+           'subscription_filter': SUBSCRIPTION_FILTER,
+           'cache_add_replicas': CACHE_ADD_REPLICAS,
+           'cache_delete_replicas': CACHE_DELETE_REPLICAS,
+           'account_attribute': ACCOUNT_ATTRIBUTE}
+
+
+def validate_schema(name, obj):
+    """
+    Validate object against json schema
+
+    :param name: The json schema name.
+    :param obj: The object to validate.
+    """
+    try:
+        if obj:
+            validate(obj, SCHEMAS.get(name, {}))
+    except ValidationError as error:  # NOQA, pylint: disable=W0612
+        raise InvalidObject("Problem validating %(name)s : %(error)s" % locals())


Test patch:
diff --git a/lib/rucio/tests/test_schema_cms.py b/lib/rucio/tests/test_schema_cms.py
old mode 100755
new mode 100644
--- a/lib/rucio/tests/test_schema_cms.py
+++ b/lib/rucio/tests/test_schema_cms.py
@@ -125,7 +125,6 @@ def test_dids(self):
 
     def test_scopes(self):
         """ CMS SCHEMA (COMMON): Test CMS scopes"""
-
         validate_schema('scope', 'cms')
         validate_schema('scope', 'user.ewv')
         validate_schema('scope', 'user.ewv2')
@@ -138,3 +137,37 @@ def test_scopes(self):
             validate_schema('scope', 'higgs')  # Not user.higgs
         with assert_raises(InvalidObject):
             validate_schema('scope', 'csm')  # Anagram
+
+    def test_attachment(self):
+        """ CMS SCHEMA (COMMON): Test CMS attachment"""
+
+        # no need to re-test did pattrens
+        dids = [{
+            'scope': 'cms', 'name': '/store/mc/Fall10/DYToMuMu_M-20_TuneZ2_7TeV-pythia6/AODSIM/START38_V12-v1/0003/C0F3344F-6EC8-DF11-8ED6-E41F13181020.root'
+        }]
+
+        dataset = '/DoubleMu/aburgmei-Run2012A_22Jan2013_v1_RHembedded_trans1_tau121_ptelec1_17elec2_8_v4/USER#1f1eee22-cdee-0f1b-271b-77a7f559e7dd'
+
+        scope = 'cms'
+
+        # RSE pattern is defined independently so we want to re-test
+        good_rses = [None, 'T2_FR_GRIF_LLR', 'T1_IT_CNAF_Disk']
+        bad_rses = ['T2_FR_GRIF-LLR', 'T4_FR_GRIF_LLR']
+
+        with assert_raises(InvalidObject):   # missing dids
+            validate_schema('attachment', {'rse': 'T1_FR_CCIN2P3'})
+
+        args = {'dids': dids}
+        validate_schema('attachment', args)
+        args['scope'] = scope
+        args['name'] = dataset
+        validate_schema('attachment', args)
+
+        for rse in good_rses:
+            args['rse'] = rse
+            validate_schema('attachment', args)
+
+        for rse in bad_rses:
+            args['rse'] = rse
+            with assert_raises(InvalidObject):
+                validate_schema('attachment', args)