Repository: rucio/rucio

Issue:
bulk updates for trace schemas
Motivation
----------

### url
Incoming traces do not have `url` set, which leads to the following error:
```
Failed validating 'type' in schema['properties']['url']:
   {'type': 'string'}

On instance['url']:
    None
```
### clientState
There's also new values that `clientState` can take up
1. SERVICE_ERROR
2. CP_TIMEOUT
3. COPY_ERROR
4. STAGEIN_ATTEMPT_FAILED
5. SourceNotFound
6. MISSINGOUTPUTFILE

### remoteSite
remoteSite currently validates against the regex:
```
{'description': 'RSE name',
'pattern': '^([A-Z0-9]+([_-][A-Z0-9]+)*)$',
'type': 'string'}
```
but incoming traces have the value set to `unknown`

### scope
for `eventType: put_sm_a` 
```
Details: 'scope' is a required property
```
or
regex pattern fails as `SCOPE_LENGTH` > 25

Modification
------------
1. add `unspecified` as a default value for url
1. Add `SERVICE_ERROR` `CP_TIMEOUT` `COPY_ERROR` `STAGEIN_ATTEMPT_FAILED` `SourceNotFound` `MISSINGOUTPUTFILE` to clientState enum
1. Change remoteSite schema to `{"type": "string"}`
1. Remove scope as a required property from `put_sm_a`, 
2. Bump `SCOPE_LENGTH` to 1000 characters
 


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/schema/generic.py b/lib/rucio/common/schema/generic.py
--- a/lib/rucio/common/schema/generic.py
+++ b/lib/rucio/common/schema/generic.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2017-2021 CERN
+# Copyright 2017-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Mayank Sharma <imptodefeat@gmail.com>, 2022
 
 from jsonschema import validate, ValidationError
 
@@ -197,7 +198,9 @@
     "description": "Client state",
     "type": "string",
     "enum": ['DONE', 'FAILED', 'PROCESSING', 'ALREADY_DONE', 'FILE_NOT_FOUND', 'FOUND_IN_PCACHE', 'DOWNLOAD_ATTEMPT',
-             'FAIL_VALIDATE', 'FOUND_ROOT', 'ServiceUnavailable']
+             'FAIL_VALIDATE', 'FOUND_ROOT', 'ServiceUnavailable', 'SERVICE_ERROR', 'CP_TIMEOUT', 'COPY_ERROR',
+             'STAGEIN_ATTEMPT_FAILED', 'SourceNotFound', 'MISSINGOUTPUTFILE', 'MD_MISMATCH', 'CHECKSUMCALCULATIONFAILURE',
+             'MISSINGINPUT', 'MISSING_INPUT']
 }
 
 RULE = {"description": "Replication rule",
diff --git a/lib/rucio/core/trace.py b/lib/rucio/core/trace.py
--- a/lib/rucio/core/trace.py
+++ b/lib/rucio/core/trace.py
@@ -39,7 +39,7 @@
 
 from rucio.common.config import config_get, config_get_int
 from rucio.common.exception import InvalidObject
-from rucio.common.schema.generic import SCOPE, RSE, UUID, TIME_ENTRY, IPv4orIPv6, CLIENT_STATE
+from rucio.common.schema.generic import UUID, TIME_ENTRY, IPv4orIPv6, CLIENT_STATE
 from rucio.core.monitor import record_counter
 
 CONFIG_COMMON_LOGLEVEL = getattr(logging, config_get('common', 'loglevel', raise_exception=False, default='DEBUG').upper())
@@ -89,7 +89,7 @@
         "eventType": {"enum": ["touch"]},
         "clientState": CLIENT_STATE,
         "account": {"type": "string"},
-        "scope": SCOPE,
+        "scope": {"type": "string"},
         "filename": {"type": "string"},
         "datasetScope": {"type": ["string", "null"]},
         "dataset": {"type": ["string", "null"]},
@@ -97,8 +97,8 @@
         "traceTimeentryUnix": {"type": "number"},
         "traceIp": IPv4orIPv6,
         "traceId": UUID,
-        "localSite": RSE,
-        "remoteSite": RSE,
+        "localSite": {"type": "string"},
+        "remoteSite": {"type": "string"},
         "usrdn": {"type": "string"},
     },
     "required": ['eventType', 'clientState', 'account', 'traceTimeentry', 'traceTimeentryUnix', 'traceIp', 'traceId']
@@ -114,10 +114,10 @@
         "clientState": CLIENT_STATE,
         "account": {"type": "string"},
         "uuid": UUID,
-        "scope": SCOPE,
+        "scope": {"type": "string"},
         "datasetScope": {"type": ["string", "null"]},
         "dataset": {"type": ["string", "null"]},
-        "remoteSite": RSE,
+        "remoteSite": {"type": "string"},
         "filesize": {"type": "number"},
         "protocol": {"type": "string"},
         "transferStart": {"type": "number"},
@@ -138,17 +138,17 @@
 }
 
 DOWNLOAD_SCHEMA = {
-    "description": "upload method",
+    "description": "download method",
     "type": "object",
     "properties": {
         "eventType": {"enum": ["download"]},
         "hostname": {"type": "string"},
         "eventVersion": {"type": "string"},
-        "localSite": RSE,
-        "remoteSite": RSE,
+        "localSite": {"type": "string"},
+        "remoteSite": {"type": "string"},
         "account": {"type": "string"},
         "uuid": UUID,
-        "scope": SCOPE,
+        "scope": {"type": "string"},
         "filename": {"type": "string"},
         "datasetScope": {"type": ["string", "null"]},
         "dataset": {"type": ["string", "null"]},
@@ -177,11 +177,11 @@
         "eventType": {"enum": ["get", "get_sm", "sm_get", "get_sm_a", "sm_get_a"]},
         "clientState": CLIENT_STATE,
         "stateReason": {"type": "string"},
-        "url": {"type": "string"},
+        "url": {"type": ["string", "null"]},
         "vo": {"type": "string"},
-        "scope": SCOPE,
+        "scope": {"type": "string"},
         "eventVersion": {"type": "string"},
-        "remoteSite": RSE,
+        "remoteSite": {"type": "string"},
         "datasetScope": {"type": "string"},
         "dataset": {"type": "string"},
         "filename": {"type": "string"},
@@ -203,11 +203,11 @@
         "eventType": {"enum": ["put_sm", "put_sm_a"]},
         "clientState": CLIENT_STATE,
         "stateReason": {"type": "string"},
-        "url": {"type": "string"},
+        "url": {"type": ["string", "null"]},
         "vo": {"type": "string"},
-        "scope": SCOPE,
+        "scope": {"type": "string"},
         "eventVersion": {"type": "string"},
-        "remoteSite": RSE,
+        "remoteSite": {"type": "string"},
         "datasetScope": {"type": "string"},
         "dataset": {"type": "string"},
         "filename": {"type": "string"},
@@ -218,12 +218,34 @@
         "traceId": UUID,
         "usrdn": {"type": "string"},
         "pq": {"type": "string"},
-        "localSite": RSE
+        "localSite": {"type": "string"}
     },
-    "required": ['eventType', 'localSite', 'eventVersion', 'uuid', 'scope',
+    "required": ['eventType', 'localSite', 'eventVersion', 'uuid',
                  'filename', 'dataset']
 }
 
+SPECIAL_SCHEMA = {
+    "description": "A special schema to capture most unsupported eventTypes",
+    "type": "object",
+    "properties": {
+        "eventType": {"enum": ["sfo2eos"]},
+        "clientState": CLIENT_STATE,
+        "account": {"type": "string"},
+        "scope": {"type": "string"},
+        "filename": {"type": "string"},
+        "datasetScope": {"type": ["string", "null"]},
+        "dataset": {"type": ["string", "null"]},
+        "traceTimeentry": TIME_ENTRY,
+        "traceTimeentryUnix": {"type": "number"},
+        "traceIp": IPv4orIPv6,
+        "traceId": UUID,
+        "localSite": {"type": "string"},
+        "remoteSite": {"type": "string"},
+        "usrdn": {"type": "string"},
+    },
+    "required": ['eventType', 'clientState', 'account', 'traceTimeentry', 'traceTimeentryUnix', 'traceIp', 'traceId']
+}
+
 SCHEMAS = {
     'touch': TOUCH_SCHEMA,
     'upload': UPLOAD_SCHEMA,
@@ -237,7 +259,8 @@
     'put_sm': PUT_SCHEMA,
     'put_sm_a': PUT_SCHEMA,
     'sm_put': PUT_SCHEMA,
-    'sm_put_a': PUT_SCHEMA
+    'sm_put_a': PUT_SCHEMA,
+    'sfo2eos': SPECIAL_SCHEMA
 }
 
 FORMAT_CHECKER = draft7_format_checker
@@ -329,12 +352,20 @@ def validate_schema(obj):
 
     :param obj: The object to validate.
 
-    :raises: ValidationError
+    :raises: InvalidObject
     """
     obj = json.loads(obj)
 
     try:
         if obj and 'eventType' in obj:
+            event_type = SCHEMAS.get(obj['eventType'].lower())
+            if not event_type:
+                validation_error = ValidationError(message=f"Trace schema for eventType {obj['eventType']} is not currently supported.")
+                validation_error.cause = "SCHEMA_NOT_FOUND"
+                raise validation_error
             validate(obj, SCHEMAS.get(obj['eventType'].lower()), format_checker=FORMAT_CHECKER)
     except ValidationError as error:
-        raise InvalidObject(error)
+        if error.cause == "SCHEMA_NOT_FOUND":
+            LOGGER.error(error)
+        else:
+            raise InvalidObject(error)


Test patch:
diff --git a/lib/rucio/tests/test_trace.py b/lib/rucio/tests/test_trace.py
--- a/lib/rucio/tests/test_trace.py
+++ b/lib/rucio/tests/test_trace.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2013-2021 CERN
+# Copyright 2013-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,12 +17,13 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2014
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2017-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2017
-# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
-# - Mayank Sharma <mayank.sharma@cern.ch>, 2021
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Mayank Sharma <imptodefeat@gmail.com>, 2022
 
 import datetime
 import time
 import uuid
+import logging
 import pytest
 import json
 from rucio.common.schema.generic import IPv4orIPv6
@@ -30,6 +31,9 @@
 from rucio.common.exception import InvalidObject
 
 
+LOGGER = logging.getLogger(__name__)
+
+
 def test_submit_trace(rest_client):
     """ TRACE (REST): submit a trace via POST """
     payload = {'uuid': str(uuid.uuid4()),  # str, because not JSON serializable
@@ -87,3 +91,14 @@ def test_trace_ip():
     for obj in invalid_obj:
         with pytest.raises(InvalidObject):
             validate_schema(obj)
+
+
+def test_non_existant_event_type_validation_rejection(caplog):
+    """
+    Test if an incoming trace with a non-supported schema logs a warning
+    instead of printing entire stack-trace
+    """
+    event_type = "put_new_type"
+    obj = json.dumps({"eventType": f"{event_type}"})
+    validate_schema(obj)
+    assert f"schema for eventType {event_type} is not currently supported" in caplog.text