Repository: rucio/rucio
Issue: add_key broken on mySQL

Motivation
----------

>>> from rucio.core.meta import add_key
>>> from rucio.db.sqla.constants import DIDType 
>>> add_key('datatype', DIDType.FILE)  
(_mysql_exceptions.DataError) (1265, "Data truncated for column 'key_type' at row 1") [SQL: u'INSERT INTO did_keys (`key`, key_type, value_type, value_regexp, updated_at, created_at) VALUES (%s, %s, %s, %s, %s, %s)'] [parameters: ('datatype', 'F', None, None, datetime.datetime(2018, 2, 19, 13, 41, 49, 985745), datetime.datetime(2018, 2, 19, 13, 41, 49, 985773))]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 358, in new_funct
    raise DatabaseException(str(error))
rucio.common.exception.DatabaseException: Database exception.
Details: (_mysql_exceptions.DataError) (1265, "Data truncated for column 'key_type' at row 1") [SQL: u'INSERT INTO did_keys (`key`, key_type, value_type, value_regexp, updated_at, created_at) VALUES (%s, %s, %s, %s, %s, %s)'] [parameters: ('datatype', 'F', None, None, datetime.datetime(2018, 2, 19, 13, 41, 49, 985745), datetime.datetime(2018, 2, 19, 13, 41, 49, 985773))]

Probably related to str vs unicode

Modification
------------

Fit it



Changed files:
lib/rucio/common/exception.py, lib/rucio/api/meta.py, lib/rucio/web/rest/webpy/v1/meta.py, lib/rucio/core/meta.py, lib/rucio/web/rest/flaskapi/v1/meta.py, lib/rucio/tests/test_meta.py

diff --git a/lib/rucio/api/meta.py b/lib/rucio/api/meta.py
index 392b5037e..e433c86ad 100644
--- a/lib/rucio/api/meta.py
+++ b/lib/rucio/api/meta.py
@@ -6,13 +6,13 @@
 #
 # Authors:
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2013
+# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
 
 from rucio.api.permission import has_permission
 from rucio.common.exception import AccessDenied
 from rucio.core import meta
-from rucio.db.sqla.constants import KeyType
 
 
 def list_keys():
@@ -49,7 +49,7 @@ def add_key(key, key_type, issuer, value_type=None, value_regexp=None):
     kwargs = {'key': key, 'key_type': key_type, 'value_type': value_type, 'value_regexp': value_regexp}
     if not has_permission(issuer=issuer, action='add_key', kwargs=kwargs):
         raise AccessDenied('Account %s can not add key' % (issuer))
-    return meta.add_key(key=key, key_type=KeyType.from_sym(key_type), value_type=value_type, value_regexp=value_regexp)
+    return meta.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp)
 
 
 def add_value(key, value, issuer):
diff --git a/lib/rucio/common/exception.py b/lib/rucio/common/exception.py
index 946d0b4ec..eb348fbb1 100644
--- a/lib/rucio/common/exception.py
+++ b/lib/rucio/common/exception.py
@@ -902,3 +902,13 @@ class RSEAttributeNotFound(RucioException):
         super(RSEAttributeNotFound, self).__init__(*args, **kwargs)
         self._message = "RSE attribute not found."
         self.error_code = 85
+
+
+class UnsupportedKeyType(RucioException):
+    """
+    RucioException
+    """
+    def __init__(self, *args, **kwargs):
+        super(UnsupportedKeyType, self).__init__(*args, **kwargs)
+        self._message = "Unsupported type for the key."
+        self.error_code = 86
diff --git a/lib/rucio/core/meta.py b/lib/rucio/core/meta.py
index a96cd14ba..643f3b16f 100644
--- a/lib/rucio/core/meta.py
+++ b/lib/rucio/core/meta.py
@@ -9,8 +9,12 @@
   Authors:
   - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2015
   - Mario Lassnig, <mario.lassnig@cern.ch>, 2013
+  - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
+
+  PY3K COMPATIBLE
 '''
 
+from __future__ import print_function
 from re import match
 
 from sqlalchemy.exc import IntegrityError
@@ -19,9 +23,9 @@ from sqlalchemy.orm.exc import NoResultFound
 from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
 from rucio.common.exception import (Duplicate, RucioException,
                                     KeyNotFound, InvalidValueForKey, UnsupportedValueType,
-                                    InvalidObject)
+                                    InvalidObject, UnsupportedKeyType)
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import DIDType
+from rucio.db.sqla.constants import DIDType, KeyType
 from rucio.db.sqla.session import read_session, transactional_session
 
 
@@ -41,6 +45,20 @@ def add_key(key, key_type, value_type=None, value_regexp=None, session=None):
     if value_type and value_type not in [str(t) for t in AUTHORIZED_VALUE_TYPES]:
         raise UnsupportedValueType('The type \\'%(value_type)s\\' is not supported for values!' % locals())
 
+    # Convert key_type
+    key_type = str(key_type)
+    if key_type == 'F':
+        key_type = 'FILE'
+    elif key_type == 'D':
+        key_type = 'DATASET'
+    elif key_type == 'C':
+        key_type = 'CONTAINER'
+
+    try:
+        key_type = KeyType.from_string(key_type)
+    except ValueError as error:
+        raise UnsupportedKeyType('The type \\'%s\\' is not supported for keys!' % str(key_type))
+
     new_key = models.DIDKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type)
     try:
         new_key.save(session=session)
@@ -151,5 +169,5 @@ def validate_meta(meta, did_type, session=None):
                 filter_by(value=meta[key]).\\
                 one()
         except NoResultFound:
-            print "The value '%s' for the key '%s' is not valid" % (meta[key], key)
+            print("The value '%s' for the key '%s' is not valid" % (meta[key], key))
             raise InvalidObject("The value '%s' for the key '%s' is not valid" % (meta[key], key))
diff --git a/lib/rucio/tests/test_meta.py b/lib/rucio/tests/test_meta.py
index cd46ed24d..ba0887205 100644
--- a/lib/rucio/tests/test_meta.py
+++ b/lib/rucio/tests/test_meta.py
@@ -7,13 +7,16 @@
 #
 # Authors:
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2013
+# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
 
-from nose.tools import assert_in, assert_is_instance, assert_true, raises
+from nose.tools import assert_in, assert_is_instance, assert_true, raises, assert_raises
 
 from rucio.client.metaclient import MetaClient
-from rucio.common.exception import InvalidValueForKey, KeyNotFound, UnsupportedValueType
+from rucio.common.exception import InvalidValueForKey, KeyNotFound, UnsupportedValueType, UnsupportedKeyType
 from rucio.common.utils import generate_uuid as uuid
-from rucio.db.sqla.constants import DIDType
+from rucio.core.meta import add_key
+from rucio.db.sqla import session, models
+from rucio.db.sqla.constants import DIDType, KeyType
 
 
 class TestMetaClient():
@@ -57,7 +60,7 @@ class TestMetaClient():
 
     @raises(InvalidValueForKey)
     def xtest_add_value_with_regexp(self):
-        """ META (CORE):  Add a new value to a key with a regexp constraint"""
+        """ META (CLIENTS):  Add a new value to a key with a regexp constraint"""
         key = 'guid' + str(uuid())[:20]
         value = str(uuid())
         # regexp for uuid
@@ -80,6 +83,58 @@ class TestMetaClient():
         value = 'value_' + str(uuid())
         self.meta_client.add_value(key="Nimportnawak", value=value)
 
-    def xtest_add_key_mysql(self):
-        """ META (CORE): Add a new key to test conversions on MySQL"""
-        self.meta_client.add_key('datatype', DIDType.FILE)
+    def test_add_key(self):
+        """ META (CLIENTS): Add a new key """
+        types = [{'type': 'FILE', 'expected': KeyType.FILE},
+                 {'type': 'ALL', 'expected': KeyType.ALL},
+                 {'type': 'COLLECTION', 'expected': KeyType.COLLECTION},
+                 {'type': 'DATASET', 'expected': KeyType.DATASET},
+                 {'type': 'D', 'expected': KeyType.DATASET},
+                 {'type': 'FILE', 'expected': KeyType.FILE},
+                 {'type': 'F', 'expected': KeyType.FILE},
+                 {'type': 'DERIVED', 'expected': KeyType.DERIVED},
+                 {'type': 'C', 'expected': KeyType.CONTAINER}]
+
+        for key_type in types:
+            key_name = 'datatype%s' % str(uuid())
+            self.meta_client.add_key(key_name, key_type['type'])
+            stored_key_type = session.get_session().query(models.DIDKey).filter_by(key=key_name).one()['key_type']
+            assert_true(stored_key_type, key_type['expected'])
+
+        with assert_raises(UnsupportedKeyType):
+            self.meta_client.add_key('datatype', 'A')
+
+
+class TestMetaCore():
+    def test_add_key(self):
+        """ META (CORE): Add a new key """
+        types = [{'type': DIDType.FILE, 'expected': KeyType.FILE},
+                 {'type': DIDType.CONTAINER, 'expected': KeyType.CONTAINER},
+                 {'type': DIDType.DATASET, 'expected': KeyType.DATASET},
+                 {'type': KeyType.ALL, 'expected': KeyType.ALL},
+                 {'type': KeyType.DERIVED, 'expected': KeyType.DERIVED},
+                 {'type': KeyType.FILE, 'expected': KeyType.FILE},
+                 {'type': KeyType.COLLECTION, 'expected': KeyType.COLLECTION},
+                 {'type': KeyType.CONTAINER, 'expected': KeyType.CONTAINER},
+                 {'type': KeyType.DATASET, 'expected': KeyType.DATASET},
+                 {'type': 'FILE', 'expected': KeyType.FILE},
+                 {'type': 'ALL', 'expected': KeyType.ALL},
+                 {'type': 'COLLECTION', 'expected': KeyType.COLLECTION},
+                 {'type': 'DATASET', 'expected': KeyType.DATASET},
+                 {'type': 'D', 'expected': KeyType.DATASET},
+                 {'type': 'FILE', 'expected': KeyType.FILE},
+                 {'type': 'F', 'expected': KeyType.FILE},
+                 {'type': 'DERIVED', 'expected': KeyType.DERIVED},
+                 {'type': 'C', 'expected': KeyType.CONTAINER}]
+
+        for key_type in types:
+            key_name = 'datatype%s' % str(uuid())
+            add_key(key_name, key_type['type'])
+            stored_key_type = session.get_session().query(models.DIDKey).filter_by(key=key_name).one()['key_type']
+            assert_true(stored_key_type, key_type['expected'])
+
+        with assert_raises(UnsupportedKeyType):
+            add_key('datatype', DIDType.ARCHIVE)
+
+        with assert_raises(UnsupportedKeyType):
+            add_key('datatype', 'A')
diff --git a/lib/rucio/web/rest/flaskapi/v1/meta.py b/lib/rucio/web/rest/flaskapi/v1/meta.py
index 50b9fec8a..3b480cc9b 100755
--- a/lib/rucio/web/rest/flaskapi/v1/meta.py
+++ b/lib/rucio/web/rest/flaskapi/v1/meta.py
@@ -17,6 +17,7 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2018
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2017
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
 
@@ -26,7 +27,7 @@ from flask import Flask, Blueprint, request, Response
 from flask.views import MethodView
 
 from rucio.api.meta import add_key, add_value, list_keys, list_values
-from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, RucioException
+from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, RucioException, UnsupportedKeyType
 from rucio.common.utils import generate_http_error_flask
 from rucio.web.rest.flaskapi.v1.common import before_request, after_request
 
@@ -91,6 +92,8 @@ class Meta(MethodView):
             return generate_http_error_flask(409, 'Duplicate', error.args[0])
         except UnsupportedValueType as error:
             return generate_http_error_flask(400, 'UnsupportedValueType', error.args[0])
+        except UnsupportedKeyType as error:
+            return generate_http_error_flask(400, 'UnsupportedKeyType', error.args[0])
         except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
         except Exception as error:
diff --git a/lib/rucio/web/rest/webpy/v1/meta.py b/lib/rucio/web/rest/webpy/v1/meta.py
index ec1584444..b6adb031a 100755
--- a/lib/rucio/web/rest/webpy/v1/meta.py
+++ b/lib/rucio/web/rest/webpy/v1/meta.py
@@ -17,6 +17,7 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2017
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
 
@@ -25,7 +26,7 @@ from logging import getLogger, StreamHandler, DEBUG
 from web import application, ctx, data, Created, InternalError, loadhook, header
 
 from rucio.api.meta import add_key, add_value, list_keys, list_values
-from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, RucioException
+from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, RucioException, UnsupportedKeyType
 from rucio.common.utils import generate_http_error
 from rucio.web.rest.common import rucio_loadhook, RucioController
 
@@ -89,6 +90,8 @@ class Meta(RucioController):
             raise generate_http_error(409, 'Duplicate', error.args[0])
         except UnsupportedValueType as error:
             raise generate_http_error(400, 'UnsupportedValueType', error.args[0])
+        except UnsupportedKeyType as error:
+            raise generate_http_error(400, 'UnsupportedKeyType', error.args[0])
         except RucioException as error:
             raise generate_http_error(500, error.__class__.__name__, error.args[0])
         except Exception as error: