Repository: rucio/rucio

Issue:
Using plugin ALL or JSON returns DataIdentifierNotFound if json metadata is not set
Motivation
----------
Using plugin ALL or JSON returns DataIdentifierNotFound if json metadata is not set, e.g.:
```
    meta = get_metadata(did['scope'], did['name'], plugin='ALL', session=session)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 49, in wrapped_f
    return Retrying(*dargs, **dkw).call(f, *args, **kw)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 206, in call
    return attempt.get(self._wrap_exception)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 247, in get
    six.reraise(self.value[0], self.value[1], self.value[2])
  File "/usr/lib/python2.7/site-packages/retrying.py", line 200, in call
    attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 309, in new_funct
    return function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/did.py", line 1255, in get_metadata
    return did_meta_plugins.get_metadata(scope, name, plugin=plugin, session=session)
  File "/usr/lib/python2.7/site-packages/rucio/core/did_meta_plugins/__init__.py", line 74, in get_metadata
    metadata = meta_handler.get_metadata(scope, name, session=session)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 49, in wrapped_f
    return Retrying(*dargs, **dkw).call(f, *args, **kw)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 206, in call
    return attempt.get(self._wrap_exception)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 247, in get
    six.reraise(self.value[0], self.value[1], self.value[2])
  File "/usr/lib/python2.7/site-packages/retrying.py", line 200, in call
    attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 309, in new_funct
    return function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/did_meta_plugins/json_meta.py", line 73, in get_metadata
    raise exception.DataIdentifierNotFound("No generic metadata found for '%(scope)s:%(name)s'" % locals())
rucio.common.exception.DataIdentifierNotFound: Data identifier not found
```


Modification
------------
In case of ALL, the exception should be caught so that at least the column metadata are reported. Regarding the exception raised for the JSON alone, shouldn't we create a new exception ?




Discussion:
`DataIdentifierNotFound` should for sure only be raised if the DataIdentifier does not exist. 
If it does, `ALL` should at least report the column one and `JSON` should probably just return an empty dict.

Resolving patch:
diff --git a/lib/rucio/core/did_meta_plugins/filter_engine.py b/lib/rucio/core/did_meta_plugins/filter_engine.py
--- a/lib/rucio/core/did_meta_plugins/filter_engine.py
+++ b/lib/rucio/core/did_meta_plugins/filter_engine.py
@@ -38,6 +38,16 @@
     "": operator.eq
 }
 
+# lookup table converting pythonic operators to oracle operators
+ORACLE_OP_MAP = {
+    operator.eq: "==",
+    operator.ne: "<>",
+    operator.gt: ">",
+    operator.lt: "<",
+    operator.ge: ">=",
+    operator.le: "<="
+}
+
 # understood date formats.
 VALID_DATE_FORMATS = (
     '%Y-%m-%d %H:%M:%S',
@@ -344,13 +354,24 @@ def create_sqla_query(self, session=None, additional_model_attributes=[], additi
                         expression = or_(expression, key.is_(None))
                 elif json_column:                                                                   # -> this key filters on the content of a json column
                     if session.bind.dialect.name == 'oracle':
-                        # Compiling for Oracle yields a UnsupportedCompilationError so fall back to support for equality only.
-                        # TODO
-                        if oper != operator.eq:
-                            raise exception.FilterEngineGenericError("Oracle implementation does not support this operator.")
-                        if isinstance(value, str) and any([char in value for char in ['*', '%']]):
-                            raise exception.FilterEngineGenericError("Oracle implementation does not support wildcards.")
-                        expression = text("json_exists(meta,'$?(@.{} == \"{}\")')".format(key, value))
+                        if isinstance(value, str) and any([char in value for char in ['*', '%']]):  # wildcards
+                            if value in ('*', '%', u'*', u'%'):                                     # match wildcard exactly == no filtering on key
+                                continue
+                            else:                                                                   # partial match with wildcard == like || notlike
+                                if oper == operator.eq:
+                                    expression = text("json_exists({},'$?(@.{} like \"{}\")')".format(json_column.key, key, value.replace('*', '%')))
+                                elif oper == operator.ne:
+                                    raise exception.FilterEngineGenericError("Oracle implementation does not support this operator.")
+                        else:
+                            try:
+                                if isinstance(value, (bool)):                                       # bool must be checked first (as bool subclass of int)
+                                    expression = text("json_exists({},'$?(@.{}.boolean() {} \"{}\")')".format(json_column.key, key, ORACLE_OP_MAP[oper], value))
+                                elif isinstance(value, (int, float)):
+                                    expression = text("json_exists({},'$?(@.{} {} {})')".format(json_column.key, key, ORACLE_OP_MAP[oper], value))
+                                else:
+                                    expression = text("json_exists({},'$?(@.{} {} \"{}\")')".format(json_column.key, key, ORACLE_OP_MAP[oper], value))
+                            except Exception as e:
+                                raise exception.FilterEngineGenericError(e)
                     else:
                         if isinstance(value, str) and any([char in value for char in ['*', '%']]):  # wildcards
                             if value in ('*', '%', u'*', u'%'):                                     # match wildcard exactly == no filtering on key
@@ -376,8 +397,6 @@ def create_sqla_query(self, session=None, additional_model_attributes=[], additi
                                     expression = oper(json_column[key].as_string(), value)
                             except Exception as e:
                                 raise exception.FilterEngineGenericError(e)
-                            if oper == operator.ne:                                                 # set .ne operator to include NULLs.
-                                expression = or_(expression, json_column[key].is_(None))
                 else:
                     raise exception.FilterEngineGenericError("Requested filter on key without model attribute, but [json_column] not set.")
 
diff --git a/lib/rucio/core/did_meta_plugins/json_meta.py b/lib/rucio/core/did_meta_plugins/json_meta.py
--- a/lib/rucio/core/did_meta_plugins/json_meta.py
+++ b/lib/rucio/core/did_meta_plugins/json_meta.py
@@ -54,7 +54,7 @@ def get_metadata(self, scope, name, session=None):
             meta = getattr(row, 'meta')
             return json_lib.loads(meta) if session.bind.dialect.name in ['oracle', 'sqlite'] else meta
         except NoResultFound:
-            raise exception.DataIdentifierNotFound("No generic metadata found for '%(scope)s:%(name)s'" % locals())
+            return {}
 
     def set_metadata(self, scope, name, key, value, recursive=False, session=None):
         self.set_metadata_bulk(scope=scope, name=name, metadata={key: value}, recursive=recursive, session=session)


Test patch:
diff --git a/lib/rucio/tests/test_filter_engine.py b/lib/rucio/tests/test_filter_engine.py
--- a/lib/rucio/tests/test_filter_engine.py
+++ b/lib/rucio/tests/test_filter_engine.py
@@ -204,7 +204,7 @@ def test_OperatorsEqualNotEqual(self, session=None):
             dids = set(dids)
             self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1)
 
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             dids = []
             q = FilterEngine('testkeyint1!=1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
                 additional_model_attributes=[
@@ -214,7 +214,7 @@ def test_OperatorsEqualNotEqual(self, session=None):
                 json_column=models.DidMeta.meta)
             dids += [did for did in q.yield_per(5)]
             dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2)
+            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 0)
 
     @read_session
     def test_OneSidedInequality(self, session=None):
@@ -253,7 +253,7 @@ def test_OneSidedInequality(self, session=None):
 
         # Plugin: JSON
         #
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             did_name = self._create_tmp_DID()
             set_metadata(scope=self.tmp_scope, name=did_name, key='testkeyint1', value=1)
 
@@ -331,7 +331,7 @@ def test_CompoundInequality(self, session=None):
 
         # Plugin: JSON
         #
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             did_name = self._create_tmp_DID()
             set_metadata(scope=self.tmp_scope, name=did_name, key='testkeyint1', value=1)
 
@@ -396,7 +396,7 @@ def test_AndGroups(self, session=None):
 
         # Plugin: JSON
         #
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             did_name1 = self._create_tmp_DID()
             did_name2 = self._create_tmp_DID()
             did_name3 = self._create_tmp_DID()
@@ -425,7 +425,7 @@ def test_AndGroups(self, session=None):
                 json_column=models.DidMeta.meta)
             dids += [did for did in q.yield_per(5)]
             dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1)     # 1
+            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 0)
 
     @read_session
     def test_OrGroups(self, session=None):
@@ -465,7 +465,7 @@ def test_OrGroups(self, session=None):
             additional_model_attributes=[models.DataIdentifier.name])
         dids += [did for did in q.yield_per(5)]
         dids = set(dids)
-        self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 0)     #
+        self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 0)
 
         dids = []
         q = FilterEngine('name = {}; name = {}; name = {}'.format(did_name1, did_name2, did_name3), model_class=models.DataIdentifier).create_sqla_query(
@@ -476,7 +476,7 @@ def test_OrGroups(self, session=None):
 
         # Plugin: JSON
         #
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             did_name1 = self._create_tmp_DID()
             did_name2 = self._create_tmp_DID()
             did_name3 = self._create_tmp_DID()
@@ -527,7 +527,7 @@ def test_OrGroups(self, session=None):
                 json_column=models.DidMeta.meta)
             dids += [did for did in q.yield_per(5)]
             dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 0)     #
+            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 0)
 
             dids = []
             q = FilterEngine('name = {}; name = {}; name = {}'.format(did_name1, did_name2, did_name3), model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
@@ -566,7 +566,7 @@ def test_AndOrGroups(self, session=None):
 
         # Plugin: JSON
         #
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             did_name1 = self._create_tmp_DID()
             did_name2 = self._create_tmp_DID()
             did_name3 = self._create_tmp_DID()
@@ -584,7 +584,7 @@ def test_AndOrGroups(self, session=None):
                 json_column=models.DidMeta.meta)
             dids += [did for did in q.yield_per(5)]
             dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 3)     # 1, 2, 3
+            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2)     # 2, 3
 
             dids = []
             q = FilterEngine('testkeyint1 = 1, testkeystr1 = test; testkeyint1 != 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
@@ -595,7 +595,7 @@ def test_AndOrGroups(self, session=None):
                 json_column=models.DidMeta.meta)
             dids += [did for did in q.yield_per(5)]
             dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2)     # 2, 3
+            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1)     # 3
 
     @read_session
     def test_BackwardsCompatibilityCreatedAfter(self, session=None):
@@ -688,7 +688,7 @@ def test_Wildcards(self, session=None):
 
         # Plugin: JSON
         #
-        if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
+        if json_implemented(session=session):
             did_name1 = self._create_tmp_DID()
             did_name2 = self._create_tmp_DID()
             did_name3 = self._create_tmp_DID()
@@ -721,27 +721,28 @@ def test_Wildcards(self, session=None):
             dids = set(dids)
             self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 4)  # 1, 2, 3, 4
 
-            dids = []
-            q = FilterEngine('testkeystr1 != *anothertest*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
-                additional_model_attributes=[
-                    models.DidMeta.scope,
-                    models.DidMeta.name
-                ],
-                json_column=models.DidMeta.meta)
-            dids += [did for did in q.yield_per(5)]
-            dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2)  # 3, 4
-
-            dids = []
-            q = FilterEngine('testkeystr1 != *test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
-                additional_model_attributes=[
-                    models.DidMeta.scope,
-                    models.DidMeta.name
-                ],
-                json_column=models.DidMeta.meta)
-            dids += [did for did in q.yield_per(5)]
-            dids = set(dids)
-            self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 0)
+            if session.bind.dialect.name != 'oracle':
+                dids = []
+                q = FilterEngine('testkeystr1 != *anothertest*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
+                    additional_model_attributes=[
+                        models.DidMeta.scope,
+                        models.DidMeta.name
+                    ],
+                    json_column=models.DidMeta.meta)
+                dids += [did for did in q.yield_per(5)]
+                dids = set(dids)
+                self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2)  # 3, 4
+
+                dids = []
+                q = FilterEngine('testkeystr1 != *test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
+                    additional_model_attributes=[
+                        models.DidMeta.scope,
+                        models.DidMeta.name
+                    ],
+                    json_column=models.DidMeta.meta)
+                dids += [did for did in q.yield_per(5)]
+                dids = set(dids)
+                self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 0)
 
 
 if __name__ == '__main__':