Repository: rucio/rucio

Issue:
Missing created_at and updated_at columns in did_meta table
Motivation
----------
Somehow the columns `created_at` and `updated_at` dont get automatically created on the table `did_meta` while using the `models.py`. 
(Found in #2449)

Modification
------------





Discussion:
Just did some testing.
The database is initialized with did_meta having the columns `created_at` and `updated_at` but then the following migration is applied and the fields are missing.
https://github.com/rucio/rucio/blob/master/lib/rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py

My suggestion would be to explicitly add the fields on the DIDMeta model and create a new migration.  
But in that case, it is enough to just add a new migration to "fix" it, since the columns are created from the models.py correctly (But dropped/added wrongly by the migration)

Resolving patch:
diff --git a/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py b/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py
@@ -0,0 +1,50 @@
+# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Aristeidis Fkiaras, <aristeidis.fkiaras@cern.ch>, 2019
+
+''' fix did_meta table missing updated_at, created_at columns '''
+
+import sqlalchemy as sa
+
+from alembic import context
+from alembic.op import add_column, drop_column
+
+
+# Alembic revision identifiers
+revision = '53b479c3cb0f'
+down_revision = '2cbee484dcf9'
+
+
+def upgrade():
+    '''
+    Upgrade the database to this revision
+    '''
+
+    if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
+        schema = context.get_context().version_table_schema if context.get_context().version_table_schema else ''
+        add_column('did_meta', sa.Column('created_at', sa.DateTime), schema=schema)
+        add_column('did_meta', sa.Column('updated_at', sa.DateTime), schema=schema)
+
+
+def downgrade():
+    '''
+    Downgrade the database to the previous revision
+    '''
+
+    if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
+        schema = context.get_context().version_table_schema if context.get_context().version_table_schema else ''
+        drop_column('did_meta', 'created_at', schema=schema)
+        drop_column('did_meta', 'updated_at', schema=schema)


Test patch:
diff --git a/lib/rucio/tests/test_did_meta.py b/lib/rucio/tests/test_did_meta.py
--- a/lib/rucio/tests/test_did_meta.py
+++ b/lib/rucio/tests/test_did_meta.py
@@ -8,6 +8,7 @@
 # Authors:
 # - Asket Agarwal, <asket.agarwal96@gmail.com>
 # - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019
+# - Aristeidis Fkiaras, <aristeidis.fkiaras@cern.ch>, 2019
 #
 # PY3K COMPATIBLE
 
@@ -15,7 +16,8 @@
 
 from rucio.client.didclient import DIDClient
 from rucio.common.utils import generate_uuid as uuid
-from rucio.common.exception import RucioException, DataIdentifierNotFound, KeyNotFound
+from rucio.common.exception import DataIdentifierNotFound, KeyNotFound
+from rucio.db.sqla.session import get_session
 
 
 class TestDidMetaClient():
@@ -25,10 +27,18 @@ def setup(self):
         self.tmp_scope = 'mock'
         self.tmp_name = 'name_%s' % uuid()
         self.did_client.add_did(scope=self.tmp_scope, name=self.tmp_name, type="DATASET")
+        self.implemented = True
+        session = get_session()
+        if session.bind.dialect.name == 'oracle':
+            oracle_version = int(session.connection().connection.version.split('.')[0])
+            if oracle_version < 12:
+                self.implemented = False
+        elif session.bind.dialect.name == 'sqlite':
+            self.implemented = False
 
     def test_add_did_meta(self):
         """ META (CLIENTS) : Adds a fully set json column to a did, updates if some keys present """
-        try:
+        if self.implemented:
             data1 = {"key1": "value_" + str(uuid()), "key2": "value_" + str(uuid()), "key3": "value_" + str(uuid())}
             self.did_client.add_did_meta(scope=self.tmp_scope, name=self.tmp_name, meta=data1)
 
@@ -52,12 +62,9 @@ def test_add_did_meta(self):
             assert_equal(len(metadata), 6)
             assert_equal(metadata["key2"], "value2")
 
-        except RucioException:
-            pass
-
     def test_delete_generic_metadata(self):
         """ META (CLIENTS) : Deletes metadata key """
-        try:
+        if self.implemented:
             data = {"key1": "value_" + str(uuid()), "key2": "value_" + str(uuid()), "key3": "value_" + str(uuid())}
             self.did_client.add_did_meta(scope=self.tmp_scope, name=self.tmp_name, meta=data)
 
@@ -69,50 +76,44 @@ def test_delete_generic_metadata(self):
             with assert_raises(KeyNotFound):
                 self.did_client.delete_did_meta(scope=self.tmp_scope, name=self.tmp_name, key="key9")
 
-        except RucioException:
-            pass
-
     def test_get_generic_metadata(self):
         """ META (CLIENTS) : Gets all metadata for the given did """
-        try:
+        if self.implemented:
             data = {"key1": "value_" + str(uuid()), "key2": "value_" + str(uuid()), "key3": "value_" + str(uuid())}
             self.did_client.add_did_meta(scope=self.tmp_scope, name=self.tmp_name, meta=data)
 
             metadata = self.did_client.get_did_meta(scope=self.tmp_scope, name=self.tmp_name)
             assert_equal(metadata, data)
 
-        except RucioException:
-            pass
-
     def test_list_dids_by_generic_meta(self):
         """ META (CLIENTS) : Get all dids matching the values of the provided metadata keys """
-        try:
+        if self.implemented:
             tmp_scope = 'mock'
             tmp_dids = []
 
-            did1 = 'name_1'
+            did1 = 'name_%s' % uuid()
             tmp_dids.append(did1)
             self.did_client.add_did(scope=tmp_scope, name=did1, type="DATASET")
             data = {"key1": "value1"}
             self.did_client.add_did_meta(scope=tmp_scope, name=did1, meta=data)
 
-            did2 = 'name_2'
+            did2 = 'name_%s' % uuid()
             tmp_dids.append(did2)
             self.did_client.add_did(scope=tmp_scope, name=did2, type="DATASET")
             data = {"key1": "value1", "key2": "value2"}
             self.did_client.add_did_meta(scope=tmp_scope, name=did2, meta=data)
 
-            did3 = 'name_3'
+            did3 = 'name_%s' % uuid()
             tmp_dids.append(did3)
             self.did_client.add_did(scope=tmp_scope, name=did3, type="DATASET")
             data = {"key1": "value1", "key2": "value2", "key3": "value3"}
             self.did_client.add_did_meta(scope=tmp_scope, name=did3, meta=data)
 
-            did4 = 'name_4'
+            did4 = 'name_%s' % uuid()
             tmp_dids.append(did4)
-            self.did_client.add_did(scope=tmp_scope, name=did1, type="DATASET")
+            self.did_client.add_did(scope=tmp_scope, name=did4, type="DATASET")
             data = {"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}
-            self.did_client.add_did_meta(scope=tmp_scope, name=did1, meta=data)
+            self.did_client.add_did_meta(scope=tmp_scope, name=did4, meta=data)
 
             dids = self.did_client.list_dids_by_meta(scope=tmp_scope, select={"key1": "value1"})
             for did in tmp_dids:
@@ -137,8 +138,6 @@ def test_list_dids_by_generic_meta(self):
             select_query = {"key1": "value1", "key2": "value2"}
             dids = self.did_client.list_dids_by_meta(scope=tmp_scope, select=select_query)
             assert_is_instance(dids, list)
-            assert_equal(len(dids), 3)
-            assert_in({'scope': 'mock', 'name': 'name_2'}, dids)
-
-        except RucioException:
-            pass
+            # Since there is no cleanup and key is not uuid, (len(dids), 3) would fail on second run.
+            # assert_equal(len(dids), 3)
+            assert_in({'scope': 'mock', 'name': did2}, dids)