Repository: privacyidea/privacyidea

Issue:
TokenOwner migration gets stuck with MariaDB
I just tried to migrate a privacyIDEA with 10000 tokens with the new migration script. The migration script ran, but at some point it did not produce any more output, but didn't terminate either.

I checked MariaDB:
```
MariaDB [(none)]> SHOW FULL PROCESSLIST;    
+----+------+-----------------+------+---------+------+---------------------------------+---------------------------------------+----------+
| Id | User | Host            | db   | Command | Time | State                           | Info                                  | Progress |
+----+------+-----------------+------+---------+------+---------------------------------+---------------------------------------+----------+
|  5 | pi   | localhost:53538 | pi   | Query   |  770 | Waiting for table metadata lock | ALTER TABLE token DROP COLUMN user_id |    0.000 |
|  6 | pi   | localhost:53542 | pi   | Sleep   |  771 |                                 | NULL                                  |    0.000 |
|  9 | root | localhost       | NULL | Query   |    0 | NULL                            | SHOW FULL PROCESSLIST                 |    0.000 |
+----+------+-----------------+------+---------+------+---------------------------------+---------------------------------------+----------+
3 rows in set (0.00 sec)
```

Apparently, MariaDB tried to issue the ``ALTER TABLE`` command to drop the ``user_id`` column, but got stuck waiting for a "table metadata lock". [SO suggests](https://stackoverflow.com/q/13148630) that MariaDB cannot acquire the lock because another connection is interfering, so I killed the other connection manually: 
```
MariaDB [(none)]> KILL 6;
Query OK, 0 rows affected (0.00 sec)
```
which caused the columns to be removed, even though the migration script then failed with the following error: 
```python
ERROR [sqlalchemy.pool.QueuePool] Exception during reset or similar
Traceback (most recent call last):
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 742, in _finalize_fairy
    fairy._reset(pool)
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 927, in _reset
    self._reset_agent.rollback()
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1697, in rollback
    self._do_rollback()
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1735, in _do_rollback
    self.connection._rollback_impl()
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 733, in _rollback_impl
    self._handle_dbapi_exception(e, None, None, None, None)
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
    util.raise_from_cause(sqlalchemy_exception, exc_info)
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 731, in _rollback_impl
    self.engine.dialect.do_rollback(self.connection)
  File "/opt/venv/lib/python2.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2148, in do_rollback
    dbapi_connection.rollback()
  File "/opt/venv/lib/python2.7/site-packages/pymysql/connections.py", line 809, in rollback
    self._read_ok_packet()
  File "/opt/venv/lib/python2.7/site-packages/pymysql/connections.py", line 773, in _read_ok_packet
    pkt = self._read_packet()
  File "/opt/venv/lib/python2.7/site-packages/pymysql/connections.py", line 1032, in _read_packet
    packet_header = self._read_bytes(4)
  File "/opt/venv/lib/python2.7/site-packages/pymysql/connections.py", line 1078, in _read_bytes
    CR.CR_SERVER_LOST, "Lost connection to MySQL server during query")
OperationalError: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query') (Background on this error at: http://sqlalche.me/e/e3q8)
```

We should investigate why the connection got stuck and try to avoid the need for manually killing the connection.


Discussion:

Resolving patch:
diff --git a/migrations/versions/48ee74b8a7c8_.py b/migrations/versions/48ee74b8a7c8_.py
--- a/migrations/versions/48ee74b8a7c8_.py
+++ b/migrations/versions/48ee74b8a7c8_.py
@@ -13,52 +13,51 @@
 
 from alembic import op
 import sqlalchemy as sa
-from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.schema import Sequence
 from sqlalchemy import orm
 from privacyidea.models import ResolverRealm, TokenRealm
 import sys
 
+Base = declarative_base()
 
-db = SQLAlchemy()
 
-
-class Realm(db.Model):
+class Realm(Base):
     __tablename__ = 'realm'
     __table_args__ = {'mysql_row_format': 'DYNAMIC'}
-    id = db.Column(db.Integer, Sequence("realm_seq"), primary_key=True,
+    id = sa.Column(sa.Integer, Sequence("realm_seq"), primary_key=True,
                    nullable=False)
-    name = db.Column(db.Unicode(255), default=u'',
+    name = sa.Column(sa.Unicode(255), default=u'',
                      unique=True, nullable=False)
-    default = db.Column(db.Boolean(), default=False)
-    option = db.Column(db.Unicode(40), default=u'')
+    default = sa.Column(sa.Boolean(), default=False)
+    option = sa.Column(sa.Unicode(40), default=u'')
 
 
-class TokenOwner(db.Model):
+class TokenOwner(Base):
     __tablename__ = 'tokenowner'
-    id = db.Column(db.Integer(), Sequence("tokenowner_seq"), primary_key=True)
-    token_id = db.Column(db.Integer(), db.ForeignKey('token.id'))
-    token = db.relationship('Token', lazy='joined', backref='token_list')
-    resolver = db.Column(db.Unicode(120), default=u'', index=True)
-    user_id = db.Column(db.Unicode(320), default=u'', index=True)
-    realm_id = db.Column(db.Integer(), db.ForeignKey('realm.id'))
-    realm = db.relationship('Realm', lazy='joined', backref='realm_list')
+    id = sa.Column(sa.Integer(), Sequence("tokenowner_seq"), primary_key=True)
+    token_id = sa.Column(sa.Integer(), sa.ForeignKey('token.id'))
+    token = orm.relationship('Token', lazy='joined', backref='token_list')
+    resolver = sa.Column(sa.Unicode(120), default=u'', index=True)
+    user_id = sa.Column(sa.Unicode(320), default=u'', index=True)
+    realm_id = sa.Column(sa.Integer(), sa.ForeignKey('realm.id'))
+    realm = orm.relationship('Realm', lazy='joined', backref='realm_list')
 
 
-class Token(db.Model):
+class Token(Base):
     __tablename__ = 'token'
     __table_args__ = {'mysql_row_format': 'DYNAMIC'}
-    id = db.Column(db.Integer, Sequence("token_seq"),
+    id = sa.Column(sa.Integer, Sequence("token_seq"),
                    primary_key=True,
                    nullable=False)
-    serial = db.Column(db.Unicode(40), default=u'',
+    serial = sa.Column(sa.Unicode(40), default=u'',
                        unique=True,
                        nullable=False,
                        index=True)
-    resolver = db.Column(db.Unicode(120), default=u'',
+    resolver = sa.Column(sa.Unicode(120), default=u'',
                         index=True)
-    resolver_type = db.Column(db.Unicode(120), default=u'')
-    user_id = db.Column(db.Unicode(320),
+    resolver_type = sa.Column(sa.Unicode(120), default=u'')
+    user_id = sa.Column(sa.Unicode(320),
                        default=u'', index=True)
 
 
@@ -85,7 +84,7 @@ def upgrade():
         session = orm.Session(bind=bind)
         # For each token, that has an owner, create a tokenowner entry
         for token in session.query(Token).filter(Token.user_id):
-            token_realms = TokenRealm.query.filter(TokenRealm.token_id == token.id).all()
+            token_realms = session.query(TokenRealm).filter(TokenRealm.token_id == token.id).all()
             realm_id = None
             if not token_realms:
                 sys.stderr.write(u"{serial!s}, {userid!s}, {resolver!s}, "
@@ -97,7 +96,7 @@ def upgrade():
                 realm_id = token_realms[0].realm_id
             elif len(token_realms) > 1:
                 # If the resolver is only contained in one realm, we fetch the realms:
-                reso_realms = ResolverRealm.query.filter(ResolverRealm.resolver == token.resolver).all()
+                reso_realms = session.query(ResolverRealm).filter(ResolverRealm.resolver == token.resolver).all()
                 if not reso_realms:
                     sys.stderr.write(u"{serial!s}, {userid!s}, {resolver!s}, "
                                      u"The token is assigned, but the assigned resolver is not "


Test patch: