Repository: rucio/rucio

Issue:
PK in history tables mostly wrong
Motivation
----------
There is an inconsistency with the PKs in history tables which has basically two reasons:
- SQLAlchemy requires each table in models.py to have a PK (Otherwise it could not be a ORM)
- In ATLAS on Oracle, we do not have PKs on History tables to save storage

This creates an inconsistency how we (developers) expect these history tables to operate between instances which run the oracle schema (without PKs) and instances where the schema is generated from models.py (With PKs)

For example, there was instances where we make MERGE INSERTS in the history tables. This clearly works in cases where there is a PK, but it doesnt work for ATLAS, as there is no PK on the table. In the other way around, multiple inserts work on the tables for ATLAS, but do not work on tables created by the models.py, due to the uniqueness of the PK.

Modification
------------
I would propose to change all models.py History table to define the PK as `mapper_args` e.g.
```
__mapper_args__ = {
        'primary_key': [scope, name, child_scope, child_name]
    }
```
This basically tells SQLAlchmey that there is a PK, to satisfy this requirement, while in reality there is None. Secondly we would have to make an upgrade script which drops all PKs in these tables. 

Communities who query these history tables frequently, just need to create indices on these tables.


Discussion:
As discussed in #1820: If there is an API/CLI to access history information, we should add an index per default. If there is no API/CLI we still might want to add one, if this information is frequently queried by operations.

Resolving patch:
diff --git a/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py b/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py
@@ -0,0 +1,84 @@
+# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Martin Barisits <martin.barisits@cern.ch>, 2019
+#
+# Topic: Correct PK and IDX for history tables
+# Revision ID: bf3baa1c1474
+# Revises: 9eb936a81eb1
+# Creation Date: 2019-01-28 16:25:57.818345
+
+from alembic.op import (create_primary_key,
+                        drop_constraint, create_index,
+                        drop_index, drop_column, add_column)
+
+from alembic import context
+
+from rucio.db.sqla.types import GUID
+
+import sqlalchemy as sa
+
+
+# revision identifiers used by alembic
+revision = 'bf3baa1c1474'       # pylint: disable=invalid-name
+down_revision = '9eb936a81eb1'  # pylint: disable=invalid-name
+
+
+def upgrade():
+    '''
+    Upgrade the database to this revision
+    '''
+    if context.get_context().dialect.name != 'sqlite':  # pylint: disable=no-member
+        # CONTENTS_HISTORY
+        drop_constraint('CONTENTS_HIST_PK', 'contents_history')
+
+        # ARCHIVE_CONTENTS_HISTORY
+        drop_constraint('ARCH_CONTENTS_HISOTRY_PK', 'archive_contents_history')
+        create_index('ARCH_CONT_HIST_IDX', 'archive_contents_history', ["scope", "name"])
+
+        # RULES_HIST_RECENT
+        drop_constraint('RULES_HIST_RECENT_PK', 'rules_hist_recent')
+        drop_column('rules_hist_recent', 'history_id')
+
+        # RULES_HISTORY
+        drop_constraint('RULES_HIST_LONGTERM_PK', 'rules_history')
+        drop_column('rules_history', 'history_id')
+
+        # MESSAGES_HISTORY
+        drop_constraint('MESSAGES_HIST_ID_PK', 'messages_history')
+
+
+def downgrade():
+    '''
+    Downgrade the database to the previous revision
+    '''
+    if context.get_context().dialect.name != 'sqlite':  # pylint: disable=no-member
+        # CONTENTS_HISTORY
+        create_primary_key('CONTENTS_HIST_PK', 'contents_history', ['scope', 'name', 'child_scope', 'child_name'])
+
+        # ARCHIVE_CONTENTS_HISTORY
+        create_primary_key('ARCH_CONTENTS_HISOTRY_PK', 'archive_contents_history', ['scope', 'name', 'child_scope', 'child_name'])
+        drop_index('ARCH_CONT_HIST_IDX', 'archive_contents_history')
+
+        # RULES_HIST_RECENT
+        add_column('rules_hist_recent', sa.Column('history_id', GUID()))
+        create_primary_key('RULES_HIST_RECENT_PK', 'rules_hist_recent', ['history_id'])
+
+        # RULES_HISTORY
+        add_column('rules_history', sa.Column('history_id', GUID()))
+        create_primary_key('RULES_HIST_LONGTERM_PK', 'rules_history', ['history_id'])
+
+        # MESSAGES_HISTORY
+        create_primary_key('MESSAGES_HIST_ID_PK', 'messages_history', ['id'])
diff --git a/lib/rucio/db/sqla/models.py b/lib/rucio/db/sqla/models.py
--- a/lib/rucio/db/sqla/models.py
+++ b/lib/rucio/db/sqla/models.py
@@ -11,7 +11,7 @@
   - Angelos Molfetas, <angelos.molfetas@cern.ch>, 2012
   - Ralph Vigne, <ralph.vigne@cern.ch>, 2013
   - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2018
-  - Martin Barisits, <martin.barisits@cern.ch>, 2013-2018
+  - Martin Barisits, <martin.barisits@cern.ch>, 2013-2019
   - Wen Guan, <wen.guan@cern.ch>, 2015
   - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019
 
@@ -72,7 +72,10 @@ def _psql_rename_type(target, connection, **kw):
 @event.listens_for(Table, "before_create")
 def _oracle_json_constraint(target, connection, **kw):
     if connection.dialect.name == 'oracle' and target.name == 'did_meta':
-        oracle_version = int(connection.connection.version.split('.')[0])
+        try:
+            oracle_version = int(connection.connection.version.split('.')[0])
+        except Exception:
+            return
         if oracle_version >= 12:
             target.append_constraint(CheckConstraint('META IS JSON', 'ORACLE_META_JSON_CHK'))
 
@@ -566,8 +569,12 @@ class ConstituentAssociationHistory(BASE, ModelBase):
     md5 = Column(String(32))
     guid = Column(GUID())
     length = Column(BigInteger)
-    _table_args = (PrimaryKeyConstraint('scope', 'name', 'child_scope', 'child_name',
-                                        name='ARCH_CONTENTS_HISOTRY_PK'), )
+    __mapper_args__ = {
+        'primary_key': [scope, name, child_scope, child_name]  # Fake primary key for SQLA
+    }
+    _table_args = (Index('ARCH_CONT_HIST_IDX', 'scope', 'name'), )
+    # _table_args = (PrimaryKeyConstraint('scope', 'name', 'child_scope', 'child_name',
+    #                                     name='ARCH_CONTENTS_HISOTRY_PK'), )
 
 
 class DataIdentifierAssociationHistory(BASE, ModelBase):
@@ -588,7 +595,7 @@ class DataIdentifierAssociationHistory(BASE, ModelBase):
     did_created_at = Column(DateTime)
     deleted_at = Column(DateTime)
     __mapper_args__ = {
-        'primary_key': [scope, name, child_scope, child_name]
+        'primary_key': [scope, name, child_scope, child_name]  # Fake primary key for SQLA
     }
     # _table_args = (PrimaryKeyConstraint('scope', 'name', 'child_scope', 'child_name', name='CONTENTS_HIST_PK'),
     _table_args = (CheckConstraint('DID_TYPE IS NOT NULL', name='CONTENTS_HIST_DID_TYPE_NN'),
@@ -862,7 +869,6 @@ class ReplicationRule(BASE, ModelBase):
 class ReplicationRuleHistoryRecent(BASE, ModelBase):
     """Represents replication rules in the recent history"""
     __tablename__ = 'rules_hist_recent'
-    history_id = Column(GUID(), default=utils.generate_uuid)
     id = Column(GUID())
     subscription_id = Column(GUID())
     account = Column(String(25))
@@ -893,15 +899,16 @@ class ReplicationRuleHistoryRecent(BASE, ModelBase):
     eol_at = Column(DateTime)
     split_container = Column(Boolean())
     meta = Column(String(4000))
-    _table_args = (PrimaryKeyConstraint('history_id', name='RULES_HIST_RECENT_PK'),  # This is only a fake PK needed by SQLAlchemy, it won't be in Oracle
-                   Index('RULES_HIST_RECENT_ID_IDX', 'id'),
+    __mapper_args__ = {
+        'primary_key': [id, locks_replicating_cnt]  # Fake primary key for SQLA
+    }
+    _table_args = (Index('RULES_HIST_RECENT_ID_IDX', 'id'),
                    Index('RULES_HIST_RECENT_SC_NA_IDX', 'scope', 'name'))
 
 
 class ReplicationRuleHistory(BASE, ModelBase):
     """Represents replication rules in the longterm history"""
     __tablename__ = 'rules_history'
-    history_id = Column(GUID(), default=utils.generate_uuid)
     id = Column(GUID())
     subscription_id = Column(GUID())
     account = Column(String(25))
@@ -932,8 +939,10 @@ class ReplicationRuleHistory(BASE, ModelBase):
     eol_at = Column(DateTime)
     split_container = Column(Boolean())
     meta = Column(String(4000))
-    _table_args = (PrimaryKeyConstraint('history_id', name='RULES_HIST_LONGTERM_PK'),  # This is only a fake PK needed by SQLAlchemy, it won't be in Oracle
-                   Index('RULES_HISTORY_SCOPENAME_IDX', 'scope', 'name'))
+    __mapper_args__ = {
+        'primary_key': [id, locks_replicating_cnt]  # Fake primary key for SQLA
+    }
+    _table_args = (Index('RULES_HISTORY_SCOPENAME_IDX', 'scope', 'name'), )
 
 
 class ReplicaLock(BASE, ModelBase):
@@ -1138,7 +1147,10 @@ class MessageHistory(BASE, ModelBase):
     id = Column(GUID())
     event_type = Column(String(1024))
     payload = Column(String(4000))
-    _table_args = (PrimaryKeyConstraint('id', name='MESSAGES_HIST_ID_PK'),)  # PK needed for SQLA only
+    __mapper_args__ = {
+        'primary_key': [id]  # Fake primary key for SQLA
+    }
+    _table_args = ()  # PrimaryKeyConstraint('id', name='MESSAGES_HIST_ID_PK'),)  # PK needed for SQLA only
 
 
 class AlembicVersion(BASE):


Test patch: