Repository: rucio/rucio

Issue:
transmogrifier misbehaving with zero subscriptions
Motivation
----------
If transmogrifier is set up to run without specifying any subscriptions, the CPU load becomes excessive.

Kronos shows similar heavy CPU loads in the absence of subscriptions.

Modification
------------

set_metadata exception handler broken
Motivation
----------

Exception from server is not properly returned to client.

[Fri Mar 16 12:12:52.483227 2018] [:error] [pid 32457:tid 140443203675904] Traceback (most recent call last):
[Fri Mar 16 12:12:52.483275 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/web/rest/did.py", line 595, in POST
[Fri Mar 16 12:12:52.483300 2018] [:error] [pid 32457:tid 140443203675904]     issuer=ctx.env.get('issuer'), recursive=recursive)
[Fri Mar 16 12:12:52.483304 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/api/did.py", line 243, in set_metadata
[Fri Mar 16 12:12:52.483306 2018] [:error] [pid 32457:tid 140443203675904]     return did.set_metadata(scope=scope, name=name, key=key, value=value, recursive=recursive)
[Fri Mar 16 12:12:52.483309 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 347, in new_funct
[Fri Mar 16 12:12:52.483311 2018] [:error] [pid 32457:tid 140443203675904]     result = function(*args, **kwargs)
[Fri Mar 16 12:12:52.483313 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib/python2.7/site-packages/rucio/core/did.py", line 1214, in set_metadata
[Fri Mar 16 12:12:52.483316 2018] [:error] [pid 32457:tid 140443203675904]     update({key: value}, synchronize_session='fetch')
[Fri Mar 16 12:12:52.483318 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 3369, in update
[Fri Mar 16 12:12:52.483329 2018] [:error] [pid 32457:tid 140443203675904]     update_op.exec_()
[Fri Mar 16 12:12:52.483332 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1326, in exec_
[Fri Mar 16 12:12:52.483334 2018] [:error] [pid 32457:tid 140443203675904]     self._do_exec()
[Fri Mar 16 12:12:52.483336 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1484, in _do_exec
[Fri Mar 16 12:12:52.483362 2018] [:error] [pid 32457:tid 140443203675904]     values = self._resolved_values
[Fri Mar 16 12:12:52.483378 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1451, in _resolved_values
[Fri Mar 16 12:12:52.483380 2018] [:error] [pid 32457:tid 140443203675904]     desc = _entity_descriptor(self.mapper, k)
[Fri Mar 16 12:12:52.483382 2018] [:error] [pid 32457:tid 140443203675904]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/base.py", line 383, in _entity_descriptor
[Fri Mar 16 12:12:52.483384 2018] [:error] [pid 32457:tid 140443203675904]     (description, key)
[Fri Mar 16 12:12:52.483387 2018] [:error] [pid 32457:tid 140443203675904] InvalidRequestError: Entity '<class 'rucio.db.sqla.models.DataIdentifier'>' has no property 'EVENTS'
[Fri Mar 16 12:12:52.483393 2018] [:error] [pid 32457:tid 140443203675904]


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -868,6 +868,8 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                 rse_schemes.append(rsemgr.select_protocol(rse_settings=rse_info[rse],
                                                                           operation='read',
                                                                           domain=domain)['scheme'])
+                        except exception.RSEProtocolNotSupported:
+                            pass  # no need to be verbose
                         except Exception:
                             print(format_exc())
 
diff --git a/lib/rucio/daemons/transmogrifier.py b/lib/rucio/daemons/transmogrifier.py
--- a/lib/rucio/daemons/transmogrifier.py
+++ b/lib/rucio/daemons/transmogrifier.py
@@ -141,7 +141,7 @@ def is_matching_subscription(subscription, did, metadata):
     return True
 
 
-def transmogrifier(bulk=5, once=False):
+def transmogrifier(bulk=5, once=False, sleep_time=60):
     """
     Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash,
     identifies the subscriptions matching the DIDs and
@@ -150,6 +150,7 @@ def transmogrifier(bulk=5, once=False):
     :param thread: Thread number at startup.
     :param bulk: The number of requests to process.
     :param once: Run only once.
+    :param sleep_time: Time between two cycles.
     """
 
     executable = ' '.join(argv)
@@ -378,14 +379,15 @@ def transmogrifier(bulk=5, once=False):
             monitor.record_counter(counters='transmogrifier.addnewrule.error', delta=1)
         if once is True:
             break
-        if tottime < 10:
-            time.sleep(10 - tottime)
+        if tottime < sleep_time:
+            logging.info(prepend_str + 'Will sleep for %s seconds' % (sleep_time - tottime))
+            time.sleep(sleep_time - tottime)
     heartbeat.die(executable, hostname, pid, hb_thread)
     logging.info(prepend_str + 'Graceful stop requested')
     logging.info(prepend_str + 'Graceful stop done')
 
 
-def run(threads=1, bulk=100, once=False):
+def run(threads=1, bulk=100, once=False, sleep_time=60):
     """
     Starts up the transmogrifier threads.
     """
@@ -396,6 +398,7 @@ def run(threads=1, bulk=100, once=False):
     else:
         logging.info('starting transmogrifier threads')
         thread_list = [threading.Thread(target=transmogrifier, kwargs={'once': once,
+                                                                       'sleep_time': sleep_time,
                                                                        'bulk': bulk}) for _ in range(0, threads)]
         [thread.start() for thread in thread_list]
         logging.info('waiting for interrupts')
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -380,8 +380,11 @@ def POST(self):
                         yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'
                     yield '  <size>' + str(rfile['bytes']) + '</size>\n'
 
-                    yield '  <glfn name="/atlas/rucio/%s:%s">' % (rfile['scope'], rfile['name'])
-                    yield '</glfn>\n'
+                    yield '  <glfn name="/%s/rucio/%s:%s"></glfn>\n' % (config_get('policy', 'schema',
+                                                                                   raise_exception=False,
+                                                                                   default='generic'),
+                                                                        rfile['scope'],
+                                                                        rfile['name'])
 
                     # TODO: deprecate this
                     if select == 'geoip':


Test patch: