Repository: rucio/rucio

Issue:
Some function based indexes missing in models.py
Motivation
----------
Some function based indexes are missing in models.py.

For example the index for `REPLICAS_STATE_IDX` is missing, while it is defined in the schema.sql for oracle. For other indices this exists. e.g. `REPLICAS_TOMBSTONE_IDX` is available, although it is function based.

Modification
------------

1. Need to check which function based indices are missing in the models.py
2. They need at least to be added (non function-based) to the models.py
3. Investigate if it is possible to add function based indices with SQLAlchemy, if yes, all of them need to be re-written.


Discussion:
# Missing indices:

- `SUBSCRIPTIONS_STATE_IDX`
- `CONTENTS_RULE_EVAL_FB_IDX`
- `REPLICAS_STATE_IDX`
- `BAD_REPLICAS_ACCOUNT_IDX`
- `RULES_INJECTSTATE_IDX`  -- not needed because other index on state
- `RULES_APPROVALSTATE_IDX` -- not needed because other index on state
- `REQUESTS_DEST_RSE_ID_IDX`
- `MESSAGES_HISTORY_ETYPE_INDX`

Resolving patch:
diff --git a/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py b/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Martin Barisits <martin.barisits@cern.ch>, 2020
+
+''' Adding missing function based indices '''
+
+from alembic import context
+from alembic.op import create_index, drop_index, drop_constraint, create_foreign_key
+
+# Alembic revision identifiers
+revision = '8ea9122275b1'
+down_revision = '50280c53117c'
+
+
+def upgrade():
+    '''
+    Upgrade the database to this revision
+    '''
+    create_index('SUBSCRIPTIONS_STATE_IDX', 'subscriptions', ['state'])
+    create_index('CONTENTS_RULE_EVAL_FB_IDX', 'contents', ['rule_evaluation'])
+    create_index('REPLICAS_STATE_IDX', 'replicas', ['state'])
+    create_index('BAD_REPLICAS_ACCOUNT_IDX', 'bad_replicas', ['account'])
+    create_index('REQUESTS_DEST_RSE_ID_IDX', 'requests', ['dest_rse_id'])
+
+
+def downgrade():
+    '''
+    Downgrade the database to the previous revision
+    '''
+
+    if context.get_context().dialect.name in ['mysql']:
+        drop_constraint('BAD_REPLICAS_ACCOUNT_FK', 'bad_replicas', type_='foreignkey')
+        drop_constraint('REQUESTS_RSES_FK', 'requests', type_='foreignkey')
+
+    drop_index('SUBSCRIPTIONS_STATE_IDX', 'subscriptions')
+    drop_index('CONTENTS_RULE_EVAL_FB_IDX', 'contents')
+    drop_index('REPLICAS_STATE_IDX', 'replicas')
+    drop_index('BAD_REPLICAS_ACCOUNT_IDX', 'bad_replicas')
+    drop_index('REQUESTS_DEST_RSE_ID_IDX', 'requests')
+
+    if context.get_context().dialect.name in ['mysql']:
+        create_foreign_key('BAD_REPLICAS_ACCOUNT_FK', 'bad_replicas', 'accounts', ['account'], ['account'])
+        create_foreign_key('REQUESTS_RSES_FK', 'requests', 'rses', ['dest_rse_id'], ['id'])
diff --git a/lib/rucio/db/sqla/models.py b/lib/rucio/db/sqla/models.py
--- a/lib/rucio/db/sqla/models.py
+++ b/lib/rucio/db/sqla/models.py
@@ -504,7 +504,8 @@ class BadReplicas(BASE, ModelBase):
                    CheckConstraint('RSE_ID IS NOT NULL', name='BAD_REPLICAS_RSE_ID_NN'),
                    ForeignKeyConstraint(['account'], ['accounts.account'], name='BAD_REPLICAS_ACCOUNT_FK'),
                    Index('BAD_REPLICAS_STATE_IDX', 'rse_id', 'state'),
-                   Index('BAD_REPLICAS_EXPIRES_AT_IDX', 'expires_at'))
+                   Index('BAD_REPLICAS_EXPIRES_AT_IDX', 'expires_at'),
+                   Index('BAD_REPLICAS_ACCOUNT_IDX', 'account'))
 
 
 class BadPFNs(BASE, ModelBase):
@@ -576,7 +577,8 @@ class DataIdentifierAssociation(BASE, ModelBase):
                    ForeignKeyConstraint(['child_scope', 'child_name'], ['dids.scope', 'dids.name'], ondelete="CASCADE", name='CONTENTS_CHILD_ID_FK'),
                    CheckConstraint('DID_TYPE IS NOT NULL', name='CONTENTS_DID_TYPE_NN'),
                    CheckConstraint('CHILD_TYPE IS NOT NULL', name='CONTENTS_CHILD_TYPE_NN'),
-                   Index('CONTENTS_CHILD_SCOPE_NAME_IDX', 'child_scope', 'child_name', 'scope', 'name'))
+                   Index('CONTENTS_CHILD_SCOPE_NAME_IDX', 'child_scope', 'child_name', 'scope', 'name'),
+                   Index('CONTENTS_RULE_EVAL_FB_IDX', 'rule_evaluation'))  # Under Oracle this is a FB index
 
 
 class ConstituentAssociation(BASE, ModelBase):
@@ -830,7 +832,8 @@ class RSEFileAssociation(BASE, ModelBase):
                    CheckConstraint('bytes IS NOT NULL', name='REPLICAS_SIZE_NN'),
                    CheckConstraint('lock_cnt IS NOT NULL', name='REPLICAS_LOCK_CNT_NN'),
                    Index('REPLICAS_TOMBSTONE_IDX', 'tombstone'),
-                   Index('REPLICAS_PATH_IDX', 'path', mysql_length=get_schema_value('NAME_LENGTH')))
+                   Index('REPLICAS_PATH_IDX', 'path', mysql_length=get_schema_value('NAME_LENGTH')),
+                   Index('REPLICAS_STATE_IDX', 'state'))  # Under Oracle this is a FB Index
 
 
 class CollectionReplica(BASE, ModelBase):
@@ -934,7 +937,7 @@ class ReplicationRule(BASE, ModelBase):
                          unique=True, mysql_length={'rse_expression': 767}),
                    Index('RULES_SCOPE_NAME_IDX', 'scope', 'name'),
                    Index('RULES_EXPIRES_AT_IDX', 'expires_at'),
-                   Index('RULES_STUCKSTATE_IDX', 'state'),  # This Index is only needed for the STUCK state
+                   Index('RULES_STUCKSTATE_IDX', 'state'),  # This Index is only needed for the STUCK state, there also is FB: RULES_INJECTSTATE_IDX, RULES_APPROVALSTATE_IDX
                    Index('RULES_CHILD_RULE_ID_IDX', 'child_rule_id'))
 
 
@@ -1117,7 +1120,8 @@ class Request(BASE, ModelBase, Versioned):
                    Index('REQUESTS_TYP_STA_UPD_IDX_OLD', 'request_type', 'state', 'updated_at'),
                    Index('REQUESTS_TYP_STA_UPD_IDX', 'request_type', 'state', 'activity'),
                    Index('REQUESTS_RULEID_IDX', 'rule_id'),
-                   Index('REQUESTS_EXTERNALID_UQ', 'external_id'))
+                   Index('REQUESTS_EXTERNALID_UQ', 'external_id'),
+                   Index('REQUESTS_DEST_RSE_ID_IDX', 'dest_rse_id'))
 
 
 class Source(BASE, ModelBase, Versioned):
@@ -1187,7 +1191,8 @@ class Subscription(BASE, ModelBase, Versioned):
                    UniqueConstraint('name', 'account', name='SUBSCRIPTIONS_NAME_ACCOUNT_UQ'),
                    ForeignKeyConstraint(['account'], ['accounts.account'], name='SUBSCRIPTIONS_ACCOUNT_FK'),
                    CheckConstraint('RETROACTIVE IS NOT NULL', name='SUBSCRIPTIONS_RETROACTIVE_NN'),
-                   CheckConstraint('ACCOUNT IS NOT NULL', name='SUBSCRIPTIONS_ACCOUNT_NN'))
+                   CheckConstraint('ACCOUNT IS NOT NULL', name='SUBSCRIPTIONS_ACCOUNT_NN'),
+                   Index('SUBSCRIPTIONS_STATE_IDX', 'state'))  # Under Oracle this is a FB index
 
 
 class Token(BASE, ModelBase):


Test patch: