Repository: privacyidea/privacyidea

Issue:
Migration fails when skipping the 3.3 version
In v3.3 the dedicated admin user was added to the policy table and the corresponding data was migrated (https://github.com/privacyidea/privacyidea/blob/master/migrations/versions/a7e91b18a460_.py).
But if the migration skips the 3.3 version, the data migration fails because the [`models.py`](https://github.com/privacyidea/privacyidea/blob/master/privacyidea/models.py) describes a different version of the database than the physical version.
By using a temporary policy table description in the migration script we can mitigate this failure.
This applies to privacyIDEA version 3.4 and up.


Discussion:

Resolving patch:
diff --git a/migrations/versions/a7e91b18a460_.py b/migrations/versions/a7e91b18a460_.py
--- a/migrations/versions/a7e91b18a460_.py
+++ b/migrations/versions/a7e91b18a460_.py
@@ -12,8 +12,31 @@
 
 from alembic import op
 import sqlalchemy as sa
-from privacyidea.models import Policy
 from sqlalchemy import orm
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+# Use table definition from the point where the changes took place. If we use
+# the table definition from models.py we might run into problems
+class Policy(Base):
+    __tablename__ = "policy"
+    __table_args__ = {'mysql_row_format': 'DYNAMIC'}
+    id = sa.Column(sa.Integer, sa.Sequence("policy_seq"), primary_key=True)
+    active = sa.Column(sa.Boolean, default=True)
+    check_all_resolvers = sa.Column(sa.Boolean, default=False)
+    name = sa.Column(sa.Unicode(64), unique=True, nullable=False)
+    scope = sa.Column(sa.Unicode(32), nullable=False)
+    action = sa.Column(sa.Unicode(2000), default=u"")
+    realm = sa.Column(sa.Unicode(256), default=u"")
+    adminrealm = sa.Column(sa.Unicode(256), default=u"")
+    adminuser = sa.Column(sa.Unicode(256), default=u"")
+    resolver = sa.Column(sa.Unicode(256), default=u"")
+    user = sa.Column(sa.Unicode(256), default=u"")
+    client = sa.Column(sa.Unicode(256), default=u"")
+    time = sa.Column(sa.Unicode(64), default=u"")
+    priority = sa.Column(sa.Integer, default=1, nullable=False)
 
 
 def upgrade():
@@ -25,12 +48,11 @@ def upgrade():
 
     # Now that we added the column in the table, we can move the "user" from admin-policies to
     # the "adminuser" column
-
+    bind = op.get_bind()
+    session = orm.Session(bind=bind)
+    pol_name = None
     try:
-        bind = op.get_bind()
-        session = orm.Session(bind=bind)
-        pol_name = None
-        for policy in session.query(Policy).filter(Policy.user != "", Policy.scope == "admin"):
+        for policy in session.query(Policy).filter(Policy.user != u"", Policy.scope == u"admin"):
             pol_name = policy.name
             # move the "user" to the "adminuser"
             policy.adminuser = policy.user
@@ -38,7 +60,8 @@ def upgrade():
         session.commit()
     except Exception as exx:
         session.rollback()
-        print("Failed to migrate column adminuser in policies due to error in policy '{0!s}'.".format(pol_name))
+        print("Failed to migrate column adminuser in policies due to error in "
+              "policy '{0!s}'.".format(pol_name))
         print(exx)
 
 
diff --git a/privacyidea/lib/auditmodules/containeraudit.py b/privacyidea/lib/auditmodules/containeraudit.py
--- a/privacyidea/lib/auditmodules/containeraudit.py
+++ b/privacyidea/lib/auditmodules/containeraudit.py
@@ -46,14 +46,15 @@ class Audit(AuditBase):
     to a list of audit modules.
     """
 
-    def __init__(self, config=None):
-        super(Audit, self).__init__(config)
+    def __init__(self, config=None, startdate=None):
+        super(Audit, self).__init__(config, startdate)
         self.name = "containeraudit"
         write_conf = self.config.get('PI_AUDIT_CONTAINER_WRITE')
         read_conf = self.config.get('PI_AUDIT_CONTAINER_READ')
         # Initialize all modules
-        self.write_modules = [get_module_class(audit_module, "Audit", "log")(config) for audit_module in write_conf]
-        self.read_module = get_module_class(read_conf, "Audit", "log")(config)
+        self.write_modules = [get_module_class(audit_module, "Audit", "log")(config, startdate)
+                              for audit_module in write_conf]
+        self.read_module = get_module_class(read_conf, "Audit", "log")(config, startdate)
         if not self.read_module.is_readable:
             log.warning(u"The specified PI_AUDIT_CONTAINER_READ {0!s} is not readable.".format(self.read_module))
 


Test patch:
diff --git a/tests/test_lib_audit.py b/tests/test_lib_audit.py
--- a/tests/test_lib_audit.py
+++ b/tests/test_lib_audit.py
@@ -413,7 +413,8 @@ def test_10_container_audit(self):
                                                          "privacyidea.lib.auditmodules.sqlaudit"],
                             "PI_AUDIT_CONTAINER_READ": "privacyidea.lib.auditmodules.sqlaudit",
                             "PI_AUDIT_NO_SIGN": True,
-                            "PI_AUDIT_SQL_URI": 'sqlite:///' + os.path.join(basedir, 'data-test.sqlite')})
+                            "PI_AUDIT_SQL_URI": 'sqlite:///' + os.path.join(basedir, 'data-test.sqlite')},
+                           startdate=None)
         self.assertFalse(a.has_data)
         a.log({"action": "something_test_30"})
         self.assertTrue(a.has_data)