Repository: rucio/rucio

Issue:
add validator for traces
Motivation
----------

The tracer server accepts any kind of json send to it and expects clients to send the information using the correct schema. Sometimes a bug in a client can lead to an invalid trace that then can break any of the systems using the traces (monitoring, Kronos, etc.)

Modification
------------

Add a validator that listens on the ActiveMQ topic and constantly checks that necessary fields are set correctly or set at all. If something doesn't match the schema send out notifications.


Discussion:
Still something we should do. Might be something to look into together with @davidpob99 😄 

Resolving patch:
diff --git a/lib/rucio/common/schema/generic.py b/lib/rucio/common/schema/generic.py
--- a/lib/rucio/common/schema/generic.py
+++ b/lib/rucio/common/schema/generic.py
@@ -22,6 +22,7 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
 from jsonschema import validate, ValidationError
 
@@ -174,6 +175,25 @@
 SPLIT_CONTAINER = {"description": "Rule split container mode",
                    "type": ["boolean", "null"]}
 
+TIME_ENTRY = {
+    "description": "Datetime, ISO 8601",
+    "type": "string",
+    "pattern": r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*$'
+}
+
+IP = {
+    "description": "Internet Protocol address v4, RFC 791",
+    "type": "string",
+    "pattern": r'^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}$'
+}
+
+CLIENT_STATE = {
+    "description": "Client state",
+    "type": "string",
+    "enum": ['DONE', 'FAILED', 'PROCESSING', 'ALREADY_DONE', 'FILE_NOT_FOUND', 'FOUND_IN_PCACHE', 'DOWNLOAD_ATTEMPT',
+             'FAIL_VALIDATE', 'FOUND_ROOT']
+}
+
 RULE = {"description": "Replication rule",
         "type": "object",
         "properties": {"dids": {"type": "array"},
diff --git a/lib/rucio/core/trace.py b/lib/rucio/core/trace.py
--- a/lib/rucio/core/trace.py
+++ b/lib/rucio/core/trace.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2013-2020 CERN
+# Copyright 2013-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,7 +19,9 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2015-2018
 # - Robert Illingworth <illingwo@fnal.gov>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Martin Barisits <martin.barisits@cern.ch>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
 """
 Core tracer module
@@ -31,8 +33,11 @@
 import socket
 
 import stomp
+from jsonschema import validate, ValidationError
 
 from rucio.common.config import config_get, config_get_int
+from rucio.common.exception import InvalidObject
+from rucio.common.schema.generic import SCOPE, RSE, UUID, TIME_ENTRY, IP, CLIENT_STATE
 from rucio.core.monitor import record_counter
 
 CONFIG_COMMON_LOGLEVEL = getattr(logging, config_get('common', 'loglevel', raise_exception=False, default='DEBUG').upper())
@@ -75,6 +80,131 @@
 PASSWORD = config_get('trace', 'password')
 VHOST = config_get('trace', 'broker_virtual_host', raise_exception=False)
 
+TOUCH_SCHEMA = {
+    "description": "touch one or more DIDs",
+    "type": "object",
+    "properties": {
+        "eventType": {"enum": ["touch"]},
+        "clientState": CLIENT_STATE,
+        "account": {"type": "string"},
+        "scope": SCOPE,
+        "filename": {"type": "string"},
+        "datasetScope": {"type": ["string", "null"]},
+        "dataset": {"type": ["string", "null"]},
+        "traceTimeentry": TIME_ENTRY,
+        "traceTimeentryUnix": {"type": "number"},
+        "traceIp": IP,
+        "traceId": UUID,
+        "localSite": RSE,
+        "remoteSite": RSE,
+        "usrdn": {"type": "string"},
+    },
+    "required": ['eventType', 'clientState', 'account', 'traceTimeentry', 'traceTimeentryUnix', 'traceIp', 'traceId']
+}
+
+UPLOAD_SCHEMA = {
+    "description": "upload method",
+    "type": "object",
+    "properties": {
+        "eventType": {"enum": ["upload"]},
+        "hostname": {"type": "string"},
+        "eventVersion": {"type": "string"},
+        "clientState": CLIENT_STATE,
+        "account": {"type": "string"},
+        "uuid": UUID,
+        "scope": SCOPE,
+        "datasetScope": {"type": ["string", "null"]},
+        "dataset": {"type": ["string", "null"]},
+        "remoteSite": RSE,
+        "filesize": {"type": "number"},
+        "protocol": {"type": "string"},
+        "transferStart": {"type": "number"},
+        "transferEnd": {"type": "number"},
+        "traceTimeentry": TIME_ENTRY,
+        "traceTimeentryUnix": {"type": "number"},
+        "traceIp": IP,
+        "traceId": UUID,
+        "vo": {"type": "string"},
+        "stateReason": {"type": "string"},
+        "filename": {"type": "string"},
+        "name": {"type": "string"},
+        "usrdn": {"type": "string"},
+    },
+    "required": ['hostname', 'account', 'eventType', 'eventVersion', 'uuid', 'scope', 'dataset',
+                 'remoteSite', 'filesize', 'protocol', 'transferStart', 'traceTimeentry', 'traceTimeentryUnix',
+                 'traceIp', 'traceId']
+}
+
+DOWNLOAD_SCHEMA = {
+    "description": "upload method",
+    "type": "object",
+    "properties": {
+        "eventType": {"enum": ["download"]},
+        "hostname": {"type": "string"},
+        "eventVersion": {"type": "string"},
+        "localSite": RSE,
+        "remoteSite": RSE,
+        "account": {"type": "string"},
+        "uuid": UUID,
+        "scope": SCOPE,
+        "filename": {"type": "string"},
+        "datasetScope": {"type": ["string", "null"]},
+        "dataset": {"type": ["string", "null"]},
+        "filesize": {"type": ["number", "null"]},
+        "clientState": CLIENT_STATE,
+        "stateReason": {"type": "string"},
+        "protocol": {"type": "string"},
+        "transferStart": {"type": "number"},
+        "transferEnd": {"type": "number"},
+        "traceTimeentry": TIME_ENTRY,
+        "traceTimeentryUnix": {"type": "number"},
+        "traceIp": IP,
+        "traceId": UUID,
+        "vo": {"type": "string"},
+        "usrdn": {"type": "string"},
+        "name": {"type": "string"},
+    },
+    "required": ['hostname', 'eventType', 'localSite', 'account', 'eventVersion', 'uuid', 'scope',
+                 'filename', 'datasetScope', 'dataset', 'filesize', 'clientState', 'stateReason']
+}
+
+GET_SCHEMA = {
+    "description": "get method, mainly sent by pilots",
+    "type": "object",
+    "properties": {
+        "eventType": {"enum": ["get", "get_sm", "sm_get", "get_sm_a", "sm_get_a"]},
+        "clientState": CLIENT_STATE,
+        "stateReason": {"type": "string"},
+        "url": {"type": "string"},
+        "vo": {"type": "string"},
+        "scope": SCOPE,
+        "eventVersion": {"type": "string"},
+        "remoteSite": RSE,
+        "datasetScope": {"type": "string"},
+        "dataset": {"type": "string"},
+        "filename": {"type": "string"},
+        "name": {"type": "string"},
+        "traceTimeentry": TIME_ENTRY,
+        "traceTimeentryUnix": {"type": "number"},
+        "traceIp": IP,
+        "traceId": UUID,
+        "usrdn": {"type": "string"},
+    },
+    "required": ['hostname', 'eventType', 'localSite', 'account', 'eventType', 'eventVersion', 'uuid', 'scope',
+                 'filename', 'datasetScope', 'dataset', 'filesize', 'clientState', 'stateReason']
+}
+
+SCHEMAS = {
+    'touch': TOUCH_SCHEMA,
+    'upload': UPLOAD_SCHEMA,
+    'download': DOWNLOAD_SCHEMA,
+    'get': GET_SCHEMA,
+    'get_sm': GET_SCHEMA,
+    'sm_get': GET_SCHEMA,
+    'get_sm_a': GET_SCHEMA,
+    'sm_get_a': GET_SCHEMA
+}
+
 logging.getLogger("stomp").setLevel(logging.CRITICAL)
 
 for broker in BROKERS_ALIAS:
@@ -107,6 +237,12 @@ def trace(payload):
     ROTATING_LOGGER.debug(report)
     t_conns = CONNS[:]
 
+    try:
+        validate_schema(report)
+    except InvalidObject as error:
+        ROTATING_LOGGER.warning("Problem validating schema: %s" % error)
+        LOGGER.warning("Problem validating schema: %s" % error)
+
     try:
         for i in range(len(t_conns)):
             try:
@@ -131,3 +267,20 @@ def trace(payload):
             LOGGER.error("Unable to connect to broker. Could not send trace: %s" % report)
     except Exception as error:
         LOGGER.error(error)
+
+
+def validate_schema(obj):
+    """
+    Validate object against json schema
+
+    :param obj: The object to validate.
+
+    :raises: ValidationError
+    """
+    obj = json.loads(obj)
+
+    try:
+        if obj and 'eventType' in obj:
+            validate(obj, SCHEMAS.get(obj['eventType'].lower()))
+    except ValidationError as error:
+        raise InvalidObject(error)


Test patch: