Repository: privacyidea/privacyidea

Issue:
Add index for timestamps in db 
Some tables contain timestamp columns which are currently not indexed and could lead to performance degradation, i.e. the `date` column in`pidea_audit` when used with the [`auditlog_age` policy](https://privacyidea.readthedocs.io/en/latest/policies/admin.html#auditlog-age).
We should add db indices for timestamp/date columns which could be used for searching:
- [ ] `pidea_audit.date`
- [ ] `usercache.timestamp`
- [ ] `authcache.first_auth`, `authcahce.last_auth`
- [ ] `challenge.timestamp`
- [ ] `monitoringstats.timestamp`
- [ ] `clientapplication.lastseen`


Discussion:

Resolving patch:
diff --git a/migrations/versions/86f40f535d7c_.py b/migrations/versions/86f40f535d7c_.py
new file mode 100644
--- /dev/null
+++ b/migrations/versions/86f40f535d7c_.py
@@ -0,0 +1,34 @@
+"""v3.8: Add indices to datetime columns
+
+Revision ID: 86f40f535d7c
+Revises: ef29ba43e290
+Create Date: 2022-05-10 16:55:28.541847
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '86f40f535d7c'
+down_revision = 'ef29ba43e290'
+
+from alembic import op
+
+
+def upgrade():
+    index_data = [('challenge', 'timestamp'),
+                  ('clientapplication', 'lastseen'),
+                  ('pidea_audit', 'date'),
+                  ('usercache', 'timestamp'),
+                  ('authcache', 'first_auth'),
+                  ('authcache', 'last_auth'),
+                  ('monitoringstats', 'timestamp')]
+    for k, v in index_data:
+        try:
+            op.create_index(op.f('ix_{0!s}_{1!s}'.format(k, v)), k, [v])
+        except Exception as exx:
+            print("Could not add index for column {1!s} in table {0!s}.".format(k, v))
+            print(exx)
+
+
+def downgrade():
+    # there should be no need to disable/remove the index when downgrading.
+    pass
diff --git a/privacyidea/models.py b/privacyidea/models.py
--- a/privacyidea/models.py
+++ b/privacyidea/models.py
@@ -1319,7 +1319,7 @@ class Challenge(MethodsMixin, db.Model):
     session = db.Column(db.Unicode(512), default=u'', quote=True, name="session")
     # The token serial number
     serial = db.Column(db.Unicode(40), default=u'', index=True)
-    timestamp = db.Column(db.DateTime, default=datetime.utcnow())
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
     expiration = db.Column(db.DateTime)
     received_count = db.Column(db.Integer(), default=0)
     otp_valid = db.Column(db.Boolean, default=False)
@@ -2514,7 +2514,7 @@ class ClientApplication(MethodsMixin, db.Model):
     ip = db.Column(db.Unicode(255), nullable=False, index=True)
     hostname = db.Column(db.Unicode(255))
     clienttype = db.Column(db.Unicode(255), nullable=False, index=True)
-    lastseen = db.Column(db.DateTime)
+    lastseen = db.Column(db.DateTime, index=True, default=datetime.utcnow())
     node = db.Column(db.Unicode(255), nullable=False)
     __table_args__ = (db.UniqueConstraint('ip',
                                           'clienttype',
@@ -2686,7 +2686,7 @@ class Audit(MethodsMixin, db.Model):
     __tablename__ = AUDIT_TABLE_NAME
     __table_args__ = {'mysql_row_format': 'DYNAMIC'}
     id = db.Column(db.Integer, Sequence("audit_seq"), primary_key=True)
-    date = db.Column(db.DateTime)
+    date = db.Column(db.DateTime, index=True)
     startdate = db.Column(db.DateTime)
     duration = db.Column(db.Interval(second_precision=6))
     signature = db.Column(db.Unicode(audit_column_length.get("signature")))
@@ -2760,7 +2760,7 @@ class UserCache(MethodsMixin, db.Model):
     used_login = db.Column(db.Unicode(64), default=u"", index=True)
     resolver = db.Column(db.Unicode(120), default=u'')
     user_id = db.Column(db.Unicode(320), default=u'', index=True)
-    timestamp = db.Column(db.DateTime)
+    timestamp = db.Column(db.DateTime, index=True)
 
     def __init__(self, username, used_login, resolver, user_id, timestamp):
         self.username = username
@@ -2774,8 +2774,8 @@ class AuthCache(MethodsMixin, db.Model):
     __tablename__ = 'authcache'
     __table_args__ = {'mysql_row_format': 'DYNAMIC'}
     id = db.Column(db.Integer, Sequence("usercache_seq"), primary_key=True)
-    first_auth = db.Column(db.DateTime)
-    last_auth = db.Column(db.DateTime)
+    first_auth = db.Column(db.DateTime, index=True)
+    last_auth = db.Column(db.DateTime, index=True)
     username = db.Column(db.Unicode(64), default=u"", index=True)
     resolver = db.Column(db.Unicode(120), default=u'', index=True)
     realm = db.Column(db.Unicode(120), default=u'', index=True)
@@ -3045,7 +3045,7 @@ class MonitoringStats(MethodsMixin, db.Model):
     id = db.Column(db.Integer, Sequence("monitoringstats_seq"),
                    primary_key=True)
     # We store this as a naive datetime in UTC
-    timestamp = db.Column(db.DateTime(False), nullable=False)
+    timestamp = db.Column(db.DateTime(False), nullable=False, index=True)
     stats_key = db.Column(db.Unicode(128), nullable=False)
     stats_value = db.Column(db.Integer, nullable=False, default=0)
 


Test patch: