Repository: rucio/rucio

Issue:
Recent sqlalchemy change broke subscription evaluation
Motivation
----------
The recent sqlalchemy change with enums broke subscription evaluation with subscriptions having a filter on the `did_type`. (E.g. `did_type=DATASET`)

Modification
------------
This needs to be fixed, and the datasets need to be re-evaluated. 




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -1480,7 +1480,7 @@ def list_dids(scope, filters, type='collection', ignore_case=False, limit=None,
         for scope, name, did_type, bytes, length in query.yield_per(5):
             yield {'scope': scope,
                    'name': name,
-                   'did_type': str(did_type),
+                   'did_type': did_type.name,
                    'bytes': bytes,
                    'length': length}
     else:
diff --git a/lib/rucio/daemons/transmogrifier/transmogrifier.py b/lib/rucio/daemons/transmogrifier/transmogrifier.py
--- a/lib/rucio/daemons/transmogrifier/transmogrifier.py
+++ b/lib/rucio/daemons/transmogrifier/transmogrifier.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2018-2020 CERN
+# Copyright 2018-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,6 +25,8 @@
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Eric Vaandering <ewv@fnal.gov>, 2020
+# - James Perry <j.perry@epcc.ed.ac.uk>, 2020
+# - Martin Barisits <martin.barisits@cern.ch>, 2021
 
 import logging
 import os
@@ -138,6 +140,16 @@ def is_matching_subscription(subscription, did, metadata):
                     break
             if not match_account:
                 return False
+        elif key == 'did_type':
+            match_did_type = False
+            if not isinstance(values, list):
+                values = [values]
+            for did_type in values:
+                if did_type == metadata['did_type'].name:
+                    match_did_type = True
+                    break
+            if not match_did_type:
+                return False
         else:
             if not isinstance(values, list):
                 values = [values, ]


Test patch:
diff --git a/lib/rucio/tests/test_subscription.py b/lib/rucio/tests/test_subscription.py
--- a/lib/rucio/tests/test_subscription.py
+++ b/lib/rucio/tests/test_subscription.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2013-2020 CERN
+# Copyright 2013-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2017
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014
-# - Martin Barisits <martin.barisits@cern.ch>, 2015-2016
+# - Martin Barisits <martin.barisits@cern.ch>, 2015-2021
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
@@ -394,3 +394,23 @@ def test_run_transmogrifier(self):
         run(threads=1, bulk=1000000, once=True)
         rules = [rule for rule in self.did_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
         assert len(rules) == 2
+
+    def test_run_transmogrifier_did_type(self):
+        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with did_type subscriptions """
+        tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
+        root = InternalAccount('root', **self.vo)
+        add_scope(tmp_scope, root)
+        subscription_name = uuid()
+        dsn = 'dataset-%s' % uuid()
+        add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root)
+
+        subid = self.sub_client.add_subscription(name=subscription_name, account='root', filter={'scope': [tmp_scope.external, ], 'pattern': 'dataset-.*', 'split_rule': True, 'did_type': ['DATASET', ]},
+                                                 replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK-POSIX|MOCK2|MOCK3', 'copies': 2, 'activity': 'Data Brokering'}],
+                                                 lifetime=None, retroactive=0, dry_run=0, comments='Ni ! Ni!', priority=1)
+        run(threads=1, bulk=1000000, once=True)
+        rules = [rule for rule in self.did_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+        assert len(rules) == 2
+        set_new_dids([{'scope': tmp_scope, 'name': dsn}, ], 1)
+        run(threads=1, bulk=1000000, once=True)
+        rules = [rule for rule in self.did_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+        assert len(rules) == 2