Repository: rucio/rucio

Issue:
Test reaper console script
Motivation
----------

The reaper console script `rucio-reaper` is not tested in the testsuite.

Modification
------------
- Add test for the reaper console script.
- Install the environnement with `python setup.py develop` in the docker env to have the generated console scripts available in the docker.
- Extend the reaper argparse method and the reaper tests to validate the argparse main method and console script.






Discussion:
Before you go through the effort, we should decide if we want to move all the bin/ scripts to console scripts, or everything back to bin/
Is there any major added benefit of the console_scripts? We had one issue where a dependency conflict was only affecting the console_scripts, not the bin/ scripts. So we should get this consistent in one way or the other.

Resolving patch:
diff --git a/lib/rucio/clis/daemons/reaper/reaper.py b/lib/rucio/clis/daemons/reaper/reaper.py
--- a/lib/rucio/clis/daemons/reaper/reaper.py
+++ b/lib/rucio/clis/daemons/reaper/reaper.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne, <vgaronne@gmail.com>, 2012-2018
+# - Vincent Garonne, <vgaronne@gmail.com>, 2012-2019
 # - Wen Guan, <wguan.icedew@gmail.com>, 2014
 # - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
 
@@ -23,6 +23,7 @@
 
 import argparse
 import signal
+import sys
 
 from rucio.daemons.reaper.reaper import run, stop
 
@@ -72,11 +73,19 @@ def get_parser():
     return parser
 
 
-def main():
+def main(argv=None):
+    """
+    The main reaper method called by the command.
 
+    :param argv: Command-line arguments. Default to  sys.argv if not set.
+    """
     signal.signal(signal.SIGTERM, stop)
+
+    if argv is None:
+        argv = sys.argv[1:]
+
     parser = get_parser()
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     try:
         run(total_workers=args.total_workers, chunk_size=args.chunk_size, greedy=args.greedy,
             once=args.run_once, scheme=args.scheme, rses=args.rses, threads_per_worker=args.threads_per_worker,
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -610,6 +610,11 @@ def inject_rule(rule_id, session=None):
     except NoResultFound:
         raise RuleNotFound('No rule with the id %s found' % (rule_id))
 
+    # Check if rule will expire in the next 5 minutes:
+    if rule.child_rule_id is None and rule.expires_at is not None and rule.expires_at < datetime.utcnow() + timedelta(seconds=300):
+        logging.info('Rule %s expiring soon, skipping', str(rule.id))
+        return
+
     # Special R2D2 container handling
     if (rule.did_type == DIDType.CONTAINER and '.r2d2_request.' in rule.name) or (rule.split_container and rule.did_type == DIDType.CONTAINER):
         logging.debug("Creating dataset rules for Split Container rule %s", str(rule.id))


Test patch:
diff --git a/lib/rucio/tests/test_judge_injector.py b/lib/rucio/tests/test_judge_injector.py
--- a/lib/rucio/tests/test_judge_injector.py
+++ b/lib/rucio/tests/test_judge_injector.py
@@ -6,7 +6,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 # Authors:
-# - Martin Barisits, <martin.barisits@cern.ch>, 2015-2016
+# - Martin Barisits, <martin.barisits@cern.ch>, 2015-2019
 
 from nose.tools import assert_raises
 
@@ -136,7 +136,7 @@ def test_add_rule_with_r2d2_container_treating(self):
             add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
             attach_dids(scope, dataset, files, 'jdoe')
             attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe')
-        rule_id = add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=100, locked=False, subscription_id=None, ask_approval=True)[0]
+        rule_id = add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=True)[0]
         approve_rule(rule_id, approver='root')
         assert(get_rule(rule_id)['state'] == RuleState.INJECT)
         rule_injector(once=True)
@@ -159,8 +159,8 @@ def test_add_rule_with_r2d2_container_treating_and_duplicate_rule(self):
             add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
             attach_dids(scope, dataset, files, 'jdoe')
             attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe')
-        add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=100, locked=False, subscription_id=None, ask_approval=False)
-        rule_id = add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=100, locked=False, subscription_id=None, ask_approval=True)[0]
+        add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=False)
+        rule_id = add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=True)[0]
         approve_rule(rule_id, approver='root')
         assert(get_rule(rule_id)['state'] == RuleState.INJECT)
         rule_injector(once=True)
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,21 +13,23 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
+# - Vincent Garonne <vgaronne@gmail.com>, 2013-2019
 # - Martin Barisits <martin.barisits@cern.ch>, 2016
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
+import nose.tools
 
-from rucio.common.utils import generate_uuid
+from rucio.common.utils import execute, generate_uuid
 from rucio.core import rse as rse_core
 from rucio.core import replica as replica_core
-from rucio.daemons.reaper.reaper import reaper
+
+from rucio.clis.daemons.reaper.reaper import main
 
 
 def test_reaper():
     """ REAPER (DAEMON): Test the reaper daemon."""
     nb_files = 30
     file_size = 2147483648  # 2G
-    for i in range(nb_files):
+    for _ in range(nb_files):
         replica_core.add_replica(rse='MOCK', scope='data13_hip',
                                  name='lfn' + generate_uuid(), bytes=file_size,
                                  account='root', adler32=None, md5=None)
@@ -36,6 +38,10 @@ def test_reaper():
     rse_core.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=10737418240)
     rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10)
 
-    rses = [rse_core.get_rse('MOCK'), ]
-    reaper(once=True, rses=rses)
-    reaper(once=True, rses=rses)
+    argv = ['--run-once', '--rses', 'MOCK']
+    main(argv)
+    # Test the rucio-reaper console script
+    cmd = 'rucio-reaper ' + ' '.join(argv)
+    exitcode, out, err = execute(cmd)
+    print(cmd, out, err)
+    nose.tools.assert_equal(exitcode, 0)