Repository: privacyidea/privacyidea

Issue:
Missing sequences when upgrading to 3.9 /SQLALchemy 1.4 and mariadb
When upgrading from 3.8 to 3.9 with a mariadb we are missing sequences.
It looks like SQLAlchemy 1.4 needs sequences.

A fresh install with 3.9 and mariadb creates sequences.

We need a migration script, that creates the sequences.

Problem: We need to create the sequences with the next id, otherwise we fail with dublicate entries.
I am adding a mockup for  a migrationscript that basically works, 

* [ ] we need to add all old tables 
* [ ] and we need to verify the next id and create the sequence accordingly.


Discussion:

Resolving patch:
diff --git a/migrations/env.py b/migrations/env.py
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -27,7 +27,7 @@ def set_database_url(config):
         # because this is what Flask-SQLAlchemy does
         if url.startswith("mysql"):
             parsed_url = make_url(url)
-            parsed_url.update_query_dict({"charset": "utf8"})
+            parsed_url = parsed_url.update_query_dict({"charset": "utf8"})
             # We need to quote the password in case it contains special chars
             parsed_url = parsed_url.set(password=quote(parsed_url.password))
             url = str(parsed_url)
diff --git a/migrations/versions/5cb310101a1f_.py b/migrations/versions/5cb310101a1f_.py
new file mode 100644
--- /dev/null
+++ b/migrations/versions/5cb310101a1f_.py
@@ -0,0 +1,83 @@
+"""v3.9: Create sequences needed for SQLAlchemy 1.4
+
+E.g. mariadb now needs sequences.
+
+Revision ID: 5cb310101a1f
+Revises: 4a0aec37e7cf
+Create Date: 2023-09-08 15:59:01.374626
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5cb310101a1f'
+down_revision = '4a0aec37e7cf'
+
+from alembic import op, context
+from sqlalchemy.schema import Sequence, CreateSequence, DropSequence
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.sql.expression import func
+from sqlalchemy.exc import OperationalError, ProgrammingError
+from privacyidea.models import db
+
+
+Session = sessionmaker()
+
+
+def upgrade():
+    migration_context = context.get_context()
+    if migration_context.dialect.supports_sequences:
+        if migration_context.dialect.name in ['mariadb', 'mysql']:
+            # setting "increment" to 0 works like auto-increment but also supports replication
+            # See <https://mariadb.com/kb/en/sequence-overview/#replication>
+            inc_value = 0
+        else:
+            inc_value = 1
+        bind = op.get_bind()
+        # We only need a read session, so we do not need a commit
+        session = Session(bind=bind)
+        # Loop over all tables defined in the current models.py
+        for tbl in db.metadata.sorted_tables:
+            # we act only on tables with an "id" column
+            if 'id' in tbl.c:
+                seq = tbl.c['id'].default
+                if isinstance(seq, Sequence):
+                    seq_name = seq.name
+                else:
+                    # do we have tables with "id" columns which isn't a sequence?
+                    print(f"Table {tbl.name} has an 'id' column which isn't a sequence!")
+                    continue
+
+                # Create the sequence with the correct next_id!
+                current_id = session.query(func.max(tbl.c.id)).one()[0] or 1
+                print(f"CurrentID in Table {tbl.name}: {current_id}")
+                try:
+                    seq = Sequence(seq_name, start=(current_id + 1), increment=inc_value)
+                    print(f" +++ Creating Sequence: {seq_name}")
+                    op.execute(CreateSequence(seq, if_not_exists=True))
+                except OperationalError as exx:
+                    if exx.orig.args[0] == 1050 or "already exists" in exx.orig.args[1]:
+                        pass
+                    else:
+                        print(exx)
+                except ProgrammingError as exx:
+                    if "already exists" in exx.orig.args[0]:
+                        pass
+                    else:
+                        print(exx)
+                except Exception as exx:
+                    print(exx)
+
+
+def downgrade():
+
+    migration_context = context.get_context()
+    if migration_context.dialect.supports_sequences and migration_context.dialect.name in ['mysql', 'mariadb']:
+        # don't remove sequences unless the db is MariaDB
+        for tbl in db.metadata.sorted_tables:
+            if 'id' in tbl.c:
+                seq = tbl.c.id.default
+                if isinstance(seq, Sequence):
+                    try:
+                        op.execute(DropSequence(seq))
+                    except Exception as exx:
+                        print(exx)
diff --git a/privacyidea/models.py b/privacyidea/models.py
--- a/privacyidea/models.py
+++ b/privacyidea/models.py
@@ -2836,7 +2836,7 @@ def __init__(self, username, used_login, resolver, user_id, timestamp):
 class AuthCache(MethodsMixin, db.Model):
     __tablename__ = 'authcache'
     __table_args__ = {'mysql_row_format': 'DYNAMIC'}
-    id = db.Column(db.Integer, Sequence("usercache_seq"), primary_key=True)
+    id = db.Column(db.Integer, Sequence("authcache_seq"), primary_key=True)
     first_auth = db.Column(db.DateTime, index=True)
     last_auth = db.Column(db.DateTime, index=True)
     username = db.Column(db.Unicode(64), default="", index=True)


Test patch: