Repository: privacyidea/privacyidea

Issue:
Specified key was too long on upgrade
I am upgrading from 2.22 to 2.23

we have mariadb 5.5

What did acutally happen?

```

File "/opt/app/privacyidea/venv/local/lib/python2.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
    raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1071, u'Specified key was too long; max key length is 767 bytes') [SQL: u'\nCREATE TABLE periodictasklastrun (\n\tid INTEGER NOT NULL AUTO_INCREMENT, \n\tperiodictask_id INTEGER, \n\tnode VARCHAR(256) NOT NULL, \n\ttimestamp DATETIME NOT NULL, \n\tPRIMARY KEY (id), \n\tCONSTRAINT ptlrix_1 UNIQUE (periodictask_id, node), \n\tFOREIGN KEY(periodictask_id) REFERENCES periodictask (id)\n)ROW_FORMAT=DYNAMIC\n\n'] (Background on this error at: http://sqlalche.me/e/2j85)

```
# Configuration

* privacyIDEA Version: upgrading from 2.22 to 2.23

* Installation method: pip

see https://wildlyinaccurate.com/mysql-specified-key-was-too-long-max-key-length-is-767-bytes/ for explanation, it affects MYSQL <5.7.7

node column should be smaller 


Discussion:
@p53 With newer mariadb versoins you need to set

    innodb_file_format=Barracuda
    innodb_large_prefix=on

in your global server config file.
@cornelinux this is quite unfortunate, it would be maybe good to add it as warning to some installation docs to prepare for DB instance upgrade before PI upgrade
You can reduce the node length to 255.
Do this in the schema in models.py and in the database if necessary.

According to #1252.

We will also change this in the model.
So can we solve this by lowering the length of ``PeriodicTaskOption.key`` and ``PeriodicTaskLastRun.node``? We could even set them to a length smaller than 255.
@p53 can you please check if 255 works for you by adapting the migration script and the model to the node lenght of 255

https://github.com/privacyidea/privacyidea/blob/master/privacyidea/models.py#L2703
You might also need to decrease the key:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/models.py#L2659

Also adapt the lenghts in this migration script:
https://github.com/privacyidea/privacyidea/blob/master/migrations/versions/2c9430cfc66b_.py

@cornelinux hi, thanks, tested, upgrade successful!
@fredreichbier in which version should be put this?
~~Hmm, if we only put this into master, users with the MariaDB setup mentioned above who upgrade to 2.23.3 will have to do manual fixes, right? This doesn't seem so good to me ... so we should probably put this into 2.23.3? :-/~~

I guess the right way to fix this would be a new migration revision which checks if the problematic tables already exist, and if not, creates them with the smaller keys? I'll put together a PR.
I would recommend a migration script like:

~~~~python
try:
   # create table with short columns
except:
   # This will not work, if the table already exist

try:
   # update table to shorten the column
except:
   # This might fail, if the column is already short
~~~~

So providing two separate actions will cover all cases.


Resolving patch:
diff --git a/migrations/env.py b/migrations/env.py
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -70,7 +70,8 @@ def run_migrations_online():
     connection = engine.connect()
     context.configure(
                 connection=connection,
-                target_metadata=target_metadata
+                target_metadata=target_metadata,
+                compare_type=True
                 )
 
     try:
diff --git a/migrations/versions/d2ae8e54b628_.py b/migrations/versions/d2ae8e54b628_.py
new file mode 100644
--- /dev/null
+++ b/migrations/versions/d2ae8e54b628_.py
@@ -0,0 +1,69 @@
+"""shorten columns used in UNIQUE constraints in periodictasklastrun, periodictaskoption
+
+Revision ID: d2ae8e54b628
+Revises: 1a0710df148b
+Create Date: 2018-10-24 11:17:40.279312
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'd2ae8e54b628'
+down_revision = '1a0710df148b'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # First, try to create the table with the shorter field sizes.
+    # This will fail if the tables already exist (which will be
+    # the case in most installations)
+    try:
+        op.create_table('periodictasklastrun',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('periodictask_id', sa.Integer(), nullable=True),
+        sa.Column('node', sa.Unicode(length=255), nullable=False),
+        sa.Column('timestamp', sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(['periodictask_id'], ['periodictask.id'], ),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('periodictask_id', 'node', name='ptlrix_1'),
+        mysql_row_format='DYNAMIC'
+        )
+        op.create_table('periodictaskoption',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('periodictask_id', sa.Integer(), nullable=True),
+        sa.Column('key', sa.Unicode(length=255), nullable=False),
+        sa.Column('value', sa.Unicode(length=2000), nullable=True),
+        sa.ForeignKeyConstraint(['periodictask_id'], ['periodictask.id'], ),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('periodictask_id', 'key', name='ptoix_1'),
+        mysql_row_format='DYNAMIC'
+        )
+        print('Successfully created shortened periodictasklastrun and periodictaskoption.')
+    except Exception as exx:
+        print('Creation of periodictasklastrun and periodictaskoption with shortened columns failed: {!r}'.format(exx))
+        print('This is expected behavior if they were already present.')
+    try:
+        op.alter_column('periodictasklastrun', 'node',
+                   existing_type=sa.VARCHAR(length=256),
+                   type_=sa.Unicode(length=255),
+                   existing_nullable=False)
+        op.alter_column('periodictaskoption', 'key',
+                   existing_type=sa.VARCHAR(length=256),
+                   type_=sa.Unicode(length=255),
+                   existing_nullable=False)
+        print('Successfully shortened columns of periodictasklastrun and periodictaskoption.')
+    except Exception as exx:
+        print('Shortening of periodictasklastrun and periodictaskoption columns failed: {!r}'.format(exx))
+        print('This is expected behavior if the columns have already been shorted.')
+
+
+def downgrade():
+    op.alter_column('periodictaskoption', 'key',
+               existing_type=sa.Unicode(length=255),
+               type_=sa.VARCHAR(length=256),
+               existing_nullable=False)
+    op.alter_column('periodictasklastrun', 'node',
+               existing_type=sa.Unicode(length=255),
+               type_=sa.VARCHAR(length=256),
+               existing_nullable=False)
diff --git a/privacyidea/models.py b/privacyidea/models.py
--- a/privacyidea/models.py
+++ b/privacyidea/models.py
@@ -2656,7 +2656,7 @@ class PeriodicTaskOption(db.Model):
     id = db.Column(db.Integer, Sequence("periodictaskopt_seq"),
                    primary_key=True)
     periodictask_id = db.Column(db.Integer, db.ForeignKey('periodictask.id'))
-    key = db.Column(db.Unicode(256), nullable=False)
+    key = db.Column(db.Unicode(255), nullable=False)
     value = db.Column(db.Unicode(2000), default=u'')
 
     __table_args__ = (db.UniqueConstraint('periodictask_id',
@@ -2700,7 +2700,7 @@ class PeriodicTaskLastRun(db.Model):
     id = db.Column(db.Integer, Sequence("periodictasklastrun_seq"),
                    primary_key=True)
     periodictask_id = db.Column(db.Integer, db.ForeignKey('periodictask.id'))
-    node = db.Column(db.Unicode(256), nullable=False)
+    node = db.Column(db.Unicode(255), nullable=False)
     timestamp = db.Column(db.DateTime(False), nullable=False)
 
     __table_args__ = (db.UniqueConstraint('periodictask_id',


Test patch: