Repository: rucio/rucio

Issue:
Add a protection against bad regular expression for suspicious files in Kronos
Motivation
----------
If the regular expression used to detect the suspicious files is bad, it can crash Kronos. Need to had a protection.

set_metadata exception handler broken
Motivation
----------

Exception from server is not properly returned to client.

[Fri Mar 16 12:12:52.483227 2018] [:error] [pid 32457:tid 140443203675904] Traceback (most recent call last):
[Fri Mar 16 12:12:52.483275 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/web/rest/did.py", line 595, in POST
[Fri Mar 16 12:12:52.483300 2018] [:error] [pid 32457:tid 140443203675904]     issuer=ctx.env.get('issuer'), recursive=recursive)
[Fri Mar 16 12:12:52.483304 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/api/did.py", line 243, in set_metadata
[Fri Mar 16 12:12:52.483306 2018] [:error] [pid 32457:tid 140443203675904]     return did.set_metadata(scope=scope, name=name, key=key, value=value, recursive=recursive)
[Fri Mar 16 12:12:52.483309 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 347, in new_funct
[Fri Mar 16 12:12:52.483311 2018] [:error] [pid 32457:tid 140443203675904]     result = function(*args, **kwargs)
[Fri Mar 16 12:12:52.483313 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/core/did.py", line 1214, in set_metadata
[Fri Mar 16 12:12:52.483316 2018] [:error] [pid 32457:tid 140443203675904]     update({key: value}, synchronize_session='fetch')
[Fri Mar 16 12:12:52.483318 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 3369, in update
[Fri Mar 16 12:12:52.483329 2018] [:error] [pid 32457:tid 140443203675904]     update_op.exec_()
[Fri Mar 16 12:12:52.483332 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1326, in exec_
[Fri Mar 16 12:12:52.483334 2018] [:error] [pid 32457:tid 140443203675904]     self._do_exec()
[Fri Mar 16 12:12:52.483336 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1484, in _do_exec
[Fri Mar 16 12:12:52.483362 2018] [:error] [pid 32457:tid 140443203675904]     values = self._resolved_values
[Fri Mar 16 12:12:52.483378 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1451, in _resolved_values
[Fri Mar 16 12:12:52.483380 2018] [:error] [pid 32457:tid 140443203675904]     desc = _entity_descriptor(self.mapper, k)
[Fri Mar 16 12:12:52.483382 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/base.py", line 383, in _entity_descriptor
[Fri Mar 16 12:12:52.483384 2018] [:error] [pid 32457:tid 140443203675904]     (description, key)
[Fri Mar 16 12:12:52.483387 2018] [:error] [pid 32457:tid 140443203675904] InvalidRequestError: Entity '<class 'rucio.db.sqla.models.DataIdentifier'>' has no property 'EVENTS'
[Fri Mar 16 12:12:52.483393 2018] [:error] [pid 32457:tid 140443203675904]


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -868,6 +868,8 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                 rse_schemes.append(rsemgr.select_protocol(rse_settings=rse_info[rse],
                                                                           operation='read',
                                                                           domain=domain)['scheme'])
+                        except exception.RSEProtocolNotSupported:
+                            pass  # no need to be verbose
                         except Exception:
                             print(format_exc())
 
diff --git a/lib/rucio/daemons/tracer/kronos.py b/lib/rucio/daemons/tracer/kronos.py
--- a/lib/rucio/daemons/tracer/kronos.py
+++ b/lib/rucio/daemons/tracer/kronos.py
@@ -139,15 +139,15 @@ def __update_atime(self):
         for report in self.__reports:
             try:
                 # Identify suspicious files
-                if self.__bad_files_patterns and report['eventType'] in ['get_sm', 'get_sm_a', 'get'] and report['clientState'] not in ['DONE', 'FOUND_ROOT', 'ALREADY_DONE']:
+                if self.__bad_files_patterns and report['eventType'] in ['get_sm', 'get_sm_a', 'get'] and 'clientState' in report and report['clientState'] not in ['DONE', 'FOUND_ROOT', 'ALREADY_DONE']:
                     for pattern in self.__bad_files_patterns:
-                        if pattern.match(report['stateReason']):
+                        if 'stateReason' in report and pattern.match(report['stateReason']):
                             reason = report['stateReason'][:255]
                             try:
                                 declare_bad_file_replicas(report['url'], reason=reason, issuer='root', status=BadFilesStatus.SUSPICIOUS)
                                 logging.info('Declare suspicious file %s with reason %s' % (report['url'], reason))
                             except Exception as error:
-                                logging.error('Failed to declare suspicious file' + error)
+                                logging.error('Failed to declare suspicious file' + str(error))
 
                 # check if scope in report. if not skip this one.
                 if 'scope' not in report:
@@ -258,7 +258,7 @@ def __update_atime(self):
         logging.info('(kronos_file) updated %d replicas' % len(replicas))
 
 
-def kronos_file(once=False, thread=0, brokers_resolved=None, dataset_queue=None):
+def kronos_file(once=False, thread=0, brokers_resolved=None, dataset_queue=None, sleep_time=60):
     """
     Main loop to consume tracer reports.
     """
@@ -281,6 +281,9 @@ def kronos_file(once=False, thread=0, brokers_resolved=None, dataset_queue=None)
             bad_files_patterns.append(re.compile(pat.strip()))
     except ConfigNotFound:
         bad_files_patterns = []
+    except Exception as error:
+        logging.error('(kronos_file) Failed to get bad_file_patterns' + str(error))
+        bad_files_patterns = []
 
     use_ssl = True
     try:
@@ -312,6 +315,7 @@ def kronos_file(once=False, thread=0, brokers_resolved=None, dataset_queue=None)
 
     sanity_check(executable='kronos-file', hostname=hostname)
     while not graceful_stop.is_set():
+        start_time = time()
         live(executable='kronos-file', hostname=hostname, pid=pid, thread=thread)
         for conn in conns:
             if not conn.is_connected():
@@ -331,7 +335,10 @@ def kronos_file(once=False, thread=0, brokers_resolved=None, dataset_queue=None)
                 else:
                     conn.connect()
                 conn.subscribe(destination=config_get('tracer-kronos', 'queue'), ack='client-individual', id=subscription_id, headers={'activemq.prefetchSize': prefetch_size})
-        sleep(1)
+        tottime = time() - start_time
+        if tottime < sleep_time:
+            logging.info('(kronos_file) Will sleep for %s seconds' % (sleep_time - tottime))
+            sleep(sleep_time - tottime)
 
     logging.info('(kronos_file) graceful stop requested')
 
@@ -345,7 +352,7 @@ def kronos_file(once=False, thread=0, brokers_resolved=None, dataset_queue=None)
     logging.info('(kronos_file) graceful stop done')
 
 
-def kronos_dataset(once=False, thread=0, dataset_queue=None):
+def kronos_dataset(once=False, thread=0, dataset_queue=None, sleep_time=60):
     logging.info('(kronos_dataset) starting')
 
     hostname = gethostname()
@@ -356,11 +363,15 @@ def kronos_dataset(once=False, thread=0, dataset_queue=None):
     start = datetime.now()
     sanity_check(executable='kronos-dataset', hostname=hostname)
     while not graceful_stop.is_set():
+        start_time = time()
         live(executable='kronos-dataset', hostname=hostname, pid=pid, thread=thread)
         if (datetime.now() - start).seconds > dataset_wait:
             __update_datasets(dataset_queue)
             start = datetime.now()
-        sleep(10)
+        tottime = time() - start_time
+        if tottime < sleep_time:
+            logging.info('(kronos_dataset) Will sleep for %s seconds' % (sleep_time - tottime))
+            sleep(sleep_time - tottime)
     # once again for the backlog
     die(executable='kronos-dataset', hostname=hostname, pid=pid, thread=thread)
     logging.info('(kronos_dataset) cleaning dataset backlog before shutdown...')
@@ -435,7 +446,7 @@ def stop(signum=None, frame=None):
     graceful_stop.set()
 
 
-def run(once=False, threads=1):
+def run(once=False, threads=1, sleep_time_datasets=60, sleep_time_files=60):
     """
     Starts up the consumer threads
     """
@@ -463,9 +474,11 @@ def run(once=False, threads=1):
     thread_list = []
     for thread in range(0, threads):
         thread_list.append(Thread(target=kronos_file, kwargs={'thread': thread,
+                                                              'sleep_time': sleep_time_files,
                                                               'brokers_resolved': brokers_resolved,
                                                               'dataset_queue': dataset_queue}))
         thread_list.append(Thread(target=kronos_dataset, kwargs={'thread': thread,
+                                                                 'sleep_time': sleep_time_datasets,
                                                                  'dataset_queue': dataset_queue}))
 
     [thread.start() for thread in thread_list]
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -380,8 +380,11 @@ def POST(self):
                         yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'
                     yield '  <size>' + str(rfile['bytes']) + '</size>\n'
 
-                    yield '  <glfn name="/atlas/rucio/%s:%s">' % (rfile['scope'], rfile['name'])
-                    yield '</glfn>\n'
+                    yield '  <glfn name="/%s/rucio/%s:%s"></glfn>\n' % (config_get('policy', 'schema',
+                                                                                   raise_exception=False,
+                                                                                   default='generic'),
+                                                                        rfile['scope'],
+                                                                        rfile['name'])
 
                     # TODO: deprecate this
                     if select == 'geoip':


Test patch: