Repository: rucio/rucio

Issue:
New psycopg2-binary release incompatible with current sqlalchemy version
Motivation
----------
Yesterday a new release (2.8) of `psycopg2-binary` was published. With this release, the tests do not run through, as exceptions are handled differently with our current version of sqlalchemy.
https://pypi.org/project/psycopg2-binary/2.8/

Modification
------------
For now we will fix `setup_rucio.py` to not install the 2.8 version thus:
`psycopg2-binary>=2.4.2<2.8'

Once we update sqlalchemy to a newer version, this can be relaxed.


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/atropos/atropos.py b/lib/rucio/daemons/atropos/atropos.py
--- a/lib/rucio/daemons/atropos/atropos.py
+++ b/lib/rucio/daemons/atropos/atropos.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2018
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
-# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
@@ -32,7 +32,7 @@
 
 from rucio.db.sqla.constants import LifetimeExceptionsState
 from rucio.common.config import config_get
-from rucio.common.exception import RuleNotFound
+from rucio.common.exception import InvalidRSEExpression, RuleNotFound
 from rucio.core import heartbeat
 import rucio.core.lifetime_exception
 from rucio.core.lock import get_dataset_locks
@@ -99,8 +99,13 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
 
                     if (rule_idx % 1000) == 0:
                         logging.info(prepend_str + '%s/%s rules processed' % (rule_idx, len(rules)))
+
                     # We compute the expected eol_at
-                    rses = parse_expression(rule.rse_expression)
+                    try:
+                        rses = parse_expression(rule.rse_expression)
+                    except InvalidRSEExpression:
+                        logging.warning(prepend_str + 'Rule %s has an RSE expression that results in an empty set: %s' % (rule.id, rule.rse_expression))
+                        continue
                     eol_at = rucio.core.lifetime_exception.define_eol(rule.scope, rule.name, rses)
                     if eol_at != rule.eol_at:
                         logging.warning(prepend_str + 'The computed eol %s differs from the one recorded %s for rule %s on %s at %s' % (eol_at, rule.eol_at, rule.id,
@@ -109,6 +114,7 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
                             update_rule(rule.id, options={'eol_at': eol_at})
                         except RuleNotFound:
                             logging.warning(prepend_str + 'Cannot find rule %s on DID %s' % (rule.id, did))
+                            continue
 
                     # Check the exceptions
                     if did in lifetime_exceptions:
@@ -124,6 +130,7 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
                                 update_rule(rule.id, options={'eol_at': lifetime_exceptions[did]})
                             except RuleNotFound:
                                 logging.warning(prepend_str + 'Cannot find rule %s on DID %s' % (rule.id, did))
+                                continue
 
                     # Now check that the new eol_at is expired
                     if eol_at and eol_at < date_check:
@@ -147,6 +154,7 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
                                 update_rule(rule.id, options=options)
                             except RuleNotFound:
                                 logging.warning(prepend_str + 'Cannot find rule %s on DID %s' % (rule.id, did))
+                                continue
             except Exception:
                 exc_type, exc_value, exc_traceback = exc_info()
                 logging.critical(''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())


Test patch: