Repository: rucio/rucio

Issue:
Reaper tests make no assertions or deletions
Motivation
----------
`test_reaper.py` and `test_reaper2.py` run the reaper, but nothing is ever deleted as the `source` column in `rse_usage` is set to `srm` when generating the test data, but (by default) this is `storage` for the RSE being used in the test. There replicas also need a tombstone set on them. As a result we never delete anything when running the reaper, but as no assertions are made the test passes.
There's also a bug in the way we decide how many files to delete (for the original reaper) where if (for example) 5 files need deleting to make the required amount of free space, we only delete 4. This is due to the order in which we check file size vs adding the replica to a list of files to be deleted in core.replica.list_unlocked_replicas.

Modification
------------
Change `source` and add tombstones to the replicas generated to allow deletion. Add some asserts to the reaper tests to make sure we delete the expected number of replicas. Change how we calculate the number of files that need reaping for the original reaper.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -28,6 +28,7 @@
 # - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White, <bjwhite@fnal.gov>, 2019
 # - Luc Goossens <luc.goossens@cern.ch>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -1635,9 +1636,9 @@ def list_unlocked_replicas(rse_id, limit, bytes=None, worker_number=None, total_
     for (scope, name, path, bytes, tombstone, state) in query.yield_per(1000):
         if state != ReplicaState.UNAVAILABLE:
 
-            total_bytes += bytes
             if tombstone != OBSOLETE and needed_space is not None and total_bytes > needed_space:
                 break
+            total_bytes += bytes
 
             total_files += 1
             if total_files > limit:
diff --git a/lib/rucio/web/ui/common/utils.py b/lib/rucio/web/ui/common/utils.py
--- a/lib/rucio/web/ui/common/utils.py
+++ b/lib/rucio/web/ui/common/utils.py
@@ -9,11 +9,16 @@
 # - Thomas Beermann, <thomas.beermann@cern.ch>, 2014-2020
 # - Ruturaj Gujar, <ruturaj.gujar23@gmail.com>, 2019
 # - Jaroslav Guenther, <jaroslav.guenther@cern.ch>, 2019-2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+#
+# PY3K COMPATIBLE
 #
 # TO-DO !!! Remove passing data with account and other params to the functions
 # catch these from the webpy input() storage object
 # will allow to remove also lines around each use of select_account_name
 
+import re
+import sys
 from json import dumps, load
 from os.path import dirname, join
 from time import time
@@ -25,6 +30,20 @@
 from rucio.common.config import config_get
 from rucio.db.sqla.constants import AccountType
 
+
+if sys.version_info > (3, 0):
+    long = int
+    unicode = str
+
+escapefunc = None
+try:
+    import html
+    escapefunc = html.escape
+except ImportError:
+    import cgi
+    escapefunc = cgi.escape
+
+
 try:
     from onelogin.saml2.auth import OneLogin_Saml2_Auth
     SAML_SUPPORT = True
@@ -54,6 +73,14 @@
         AUTH_ISSUERS = []
 
 
+# excluded characters for injected JavaScript variables
+VARIABLE_VALUE_REGEX = re.compile(r"^[\w\- /=,.+*#()\[\]]*$", re.UNICODE)
+
+
+def html_escape(s, quote=True):
+    return escapefunc(s, quote)
+
+
 def prepare_webpy_request(request, data):
     """
     Prepare a webpy request for SAML
@@ -104,6 +131,10 @@ def __to_js(var, value):
     :param var: The name of the javascript var.
     :param value: The value to set.
     """
+    value = value.replace('\n', ' ')  # replace newlines for pattern matching on the whole string
+    if not VARIABLE_VALUE_REGEX.match(value):
+        # ensure nothing is injected
+        value = ''
     return '<script type="text/javascript">var %s = "%s";</script>' % (var, value)
 
 
@@ -255,7 +286,7 @@ def x509token_auth(data=None):
         ui_account = data.account
     else:
         ui_account = select_account_name(dn, 'x509')
-    msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s." % (dn, ui_account)
+    msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s." % (html_escape(dn), html_escape(ui_account))
     msg += "<br><br><font color=\"red\">First, please make sure it is correctly registered in <a href=\"https://voms2.cern.ch:8443/voms/atlas\">VOMS</a> and be patient until it has been fully propagated through the system.</font>"
     msg += "<br><br>Then, if it is still not working please contact <a href=\"mailto:atlas-adc-ddm-support@cern.ch\">DDM Support</a>."
     if not ui_account:
@@ -284,10 +315,10 @@ def userpass_auth(data, rendered_tpl):
         else:
             ui_account = select_account_name(data.username, 'userpass')
         if not ui_account:
-            return RENDERER.problem(('Cannot get find any account associated with %s identity.' % (data.username)))
+            return RENDERER.problem(('Cannot get find any account associated with %s identity.' % (html_escape(data.username))))
         token = get_token(auth.get_auth_token_user_pass, acc=ui_account, idt=data.username, pwd=data.password.encode("ascii"))
         if not token:
-            return RENDERER.problem(('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (data.username, ui_account))
+            return RENDERER.problem(('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html_escape(data.username), html_escape(ui_account)))
     return finalize_auth(token, 'userpass')
 
 
@@ -315,10 +346,10 @@ def saml_auth(method, data=None):
         if not ui_account:
             ui_account = select_account_name(saml_nameid, 'saml')
         if not ui_account:
-            return RENDERER.problem('Cannot get find any account associated with %s identity.' % (saml_nameid))
+            return RENDERER.problem('Cannot get find any account associated with %s identity.' % (html_escape(saml_nameid)))
         token = get_token(auth.get_auth_token_saml, acc=ui_account, idt=saml_nameid)
         if not token:
-            return RENDERER.problem(('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (saml_nameid, ui_account))
+            return RENDERER.problem(('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html_escape(saml_nameid), html_escape(ui_account)))
         return finalize_auth(token, 'saml')
 
     # If method is POST, check the received SAML response and redirect to home if valid
@@ -335,10 +366,10 @@ def saml_auth(method, data=None):
             if not ui_account:
                 ui_account = select_account_name(saml_nameid, 'saml')
             if not ui_account:
-                return RENDERER.problem('Cannot get find any account associated with %s identity.' % (saml_nameid))
+                return RENDERER.problem('Cannot get find any account associated with %s identity.' % (html_escape(saml_nameid)))
             token = get_token(auth.get_auth_token_saml, acc=ui_account, idt=saml_nameid)
             if not token:
-                return RENDERER.problem(('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (saml_nameid, ui_account))
+                return RENDERER.problem(('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html_escape(saml_nameid), html_escape(ui_account)))
             return finalize_auth(token, 'saml', cookie_extra)
 
         return RENDERER.problem("Not authenticated")


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -18,29 +18,56 @@
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2019
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
+
+from datetime import datetime, timedelta
+from nose.tools import assert_equal
 
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid
 from rucio.core import rse as rse_core
 from rucio.core import replica as replica_core
 from rucio.daemons.reaper.reaper import reaper
+from rucio.tests.common import rse_name_generator
 
 
 def test_reaper():
     """ REAPER (DAEMON): Test the reaper daemon."""
+    rse_name = rse_name_generator()
+    rse_id = rse_core.add_rse(rse_name)
+
+    mock_protocol = {'scheme': 'MOCK',
+                     'hostname': 'localhost',
+                     'port': 123,
+                     'prefix': '/test/reaper',
+                     'impl': 'rucio.rse.protocols.mock.Default',
+                     'domains': {
+                         'lan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1},
+                         'wan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1}}}
+    rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
+
     nb_files = 30
     file_size = 2147483648  # 2G
-    rse_id = rse_core.get_rse_id(rse='MOCK')
 
+    file_names = []
     for i in range(nb_files):
+        file_name = 'lfn' + generate_uuid()
+        file_names.append(file_name)
         replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip'),
-                                 name='lfn' + generate_uuid(), bytes=file_size,
+                                 name=file_name, bytes=file_size,
+                                 tombstone=datetime.utcnow() - timedelta(days=1),
                                  account=InternalAccount('root'), adler32=None, md5=None)
 
-    rse_core.set_rse_usage(rse_id=rse_id, source='srm', used=nb_files * file_size, free=800)
+    rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
     rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
     rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
 
-    rses = [rse_core.get_rse(rse_core.get_rse_id('MOCK')), ]
+    rses = [rse_core.get_rse(rse_id), ]
     reaper(once=True, rses=rses)
     reaper(once=True, rses=rses)
+    assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip'), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 10)
diff --git a/lib/rucio/tests/test_reaper2.py b/lib/rucio/tests/test_reaper2.py
--- a/lib/rucio/tests/test_reaper2.py
+++ b/lib/rucio/tests/test_reaper2.py
@@ -14,28 +14,55 @@
 #
 # Authors:
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
+
+from datetime import datetime, timedelta
+from nose.tools import assert_equal
 
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid
 from rucio.core import rse as rse_core
 from rucio.core import replica as replica_core
 from rucio.daemons.reaper.reaper2 import reaper
+from rucio.tests.common import rse_name_generator
 
 
 def test_reaper():
     """ REAPER2 (DAEMON): Test the reaper2 daemon."""
+    rse_name = rse_name_generator()
+    rse_id = rse_core.add_rse(rse_name)
+
+    mock_protocol = {'scheme': 'MOCK',
+                     'hostname': 'localhost',
+                     'port': 123,
+                     'prefix': '/test/reaper',
+                     'impl': 'rucio.rse.protocols.mock.Default',
+                     'domains': {
+                         'lan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1},
+                         'wan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1}}}
+    rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
+
     nb_files = 30
     file_size = 2147483648  # 2G
-    rse_id = rse_core.get_rse_id(rse='MOCK')
 
+    file_names = []
     for i in range(nb_files):
+        file_name = 'lfn' + generate_uuid()
+        file_names.append(file_name)
         replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip'),
-                                 name='lfn' + generate_uuid(), bytes=file_size,
+                                 name=file_name, bytes=file_size,
+                                 tombstone=datetime.utcnow() - timedelta(days=1),
                                  account=InternalAccount('root'), adler32=None, md5=None)
 
-    rse_core.set_rse_usage(rse_id=rse_id, source='srm', used=nb_files * file_size, free=800)
+    rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
     rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
     rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
 
-    reaper(once=True, rses=[], include_rses='MOCK', exclude_rses=[])
-    reaper(once=True, rses=[], include_rses='MOCK', exclude_rses=[])
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
+
+    assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip'), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 5)