Repository: adamchainz/django-mysql

Issue:
Migration Operations - Change Storage Engine
MySQL's storage engines are underrepresented in Django. A migration operation to change the storage engine from X to Y would be helpful for e.g. changing a table's transaction support (MyISAM <-> InnoDB), or moving to MariaDB (InnoDB -> XtraDB).

Maybe storage engines can be grafted onto `Model.Meta` even?



Discussion:

Resolving patch:
diff --git a/django_mysql/operations.py b/django_mysql/operations.py
--- a/django_mysql/operations.py
+++ b/django_mysql/operations.py
@@ -1,4 +1,5 @@
 from django.db.migrations.operations.base import Operation
+from django.utils.functional import cached_property
 
 
 class InstallPlugin(Operation):
@@ -58,3 +59,85 @@ def database_backwards(self, app_label, schema_editor, from_st, to_st):
 
     def describe(self):
         return "Installs library %s" % (self.soname)
+
+
+class AlterStorageEngine(Operation):
+
+    def __init__(self, name, to_engine, from_engine=None):
+        self.name = name
+        self.engine = to_engine
+        self.from_engine = from_engine
+
+    @property
+    def reversible(self):
+        return (self.from_engine is not None)
+
+    def state_forwards(self, app_label, state):
+        pass
+
+    def database_forwards(self, app_label, schema_editor, from_state, to_state,
+                          engine=None):
+        if engine is None:
+            engine = self.engine
+
+        if hasattr(to_state, 'render'):
+            apps = to_state.render()  # Django 1.7
+        else:
+            apps = to_state.apps  # Django 1.8+
+
+        new_model = apps.get_model(app_label, self.name)
+
+        if hasattr(self, 'allowed_to_migrate'):
+            allowed = self.allowed_to_migrate  # Django 1.7
+        else:
+            allowed = self.allow_migrate_model  # Django 1.8+
+
+        qn = schema_editor.connection.ops.quote_name
+
+        if allowed(schema_editor.connection.alias, new_model):
+            with schema_editor.connection.cursor() as cursor:
+                cursor.execute(
+                    """SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
+                       WHERE TABLE_SCHEMA=DATABASE() AND
+                             TABLE_NAME = %s AND
+                             ENGINE = %s""",
+                    (new_model._meta.db_table, engine)
+                )
+                uses_engine_already = (cursor.fetchone()[0] > 0)
+
+            if uses_engine_already:
+                return
+
+            schema_editor.execute(
+                "ALTER TABLE {table} ENGINE={engine}".format(
+                    table=qn(new_model._meta.db_table),
+                    engine=engine
+                )
+            )
+
+    def database_backwards(self, app_label, schema_editor, from_state,
+                           to_state):
+        if self.from_engine is None:
+            raise NotImplementedError("You cannot reverse this operation")
+
+        return self.database_forwards(app_label, schema_editor, from_state,
+                                      to_state, engine=self.from_engine)
+
+    @cached_property
+    def name_lower(self):
+        return self.name.lower()
+
+    def references_model(self, name, app_label=None):
+        return name.lower() == self.name_lower
+
+    def describe(self):
+        if self.from_engine:
+            from_clause = " from {}".format(self.from_engine)
+        else:
+            from_clause = ""
+        return "Alter storage engine for {model}{from_clause} to {engine}" \
+               .format(
+                   model=self.name,
+                   from_clause=from_clause,
+                   engine=self.engine
+               )


Test patch:
diff --git a/tests/django_mysql_tests/test_operations.py b/tests/django_mysql_tests/test_operations.py
--- a/tests/django_mysql_tests/test_operations.py
+++ b/tests/django_mysql_tests/test_operations.py
@@ -1,17 +1,21 @@
 from unittest import SkipTest
 
-from django.db import connection
+from django.db import connection, migrations, models, transaction
 from django.db.migrations.state import ProjectState
 from django.test import TransactionTestCase
+from django.test.utils import CaptureQueriesContext
 
-from django_mysql.operations import InstallPlugin, InstallSOName
+from django_mysql.operations import (
+    AlterStorageEngine, InstallPlugin, InstallSOName
+)
+from django_mysql.test.utils import override_mysql_variables
 
 
-class OperationTests(TransactionTestCase):
+class PluginOperationTests(TransactionTestCase):
 
     @classmethod
     def setUpClass(cls):
-        super(OperationTests, cls).setUpClass()
+        super(PluginOperationTests, cls).setUpClass()
         has_metadata_lock_plugin = (
             (connection.is_mariadb and connection.mysql_version >= (10, 0, 7))
         )
@@ -87,3 +91,211 @@ def plugin_count(self, plugin_name):
                 (plugin_name,)
             )
             return cursor.fetchone()[0]
+
+
+class AlterStorageEngineTests(TransactionTestCase):
+
+    def test_no_from_means_unreversible(self):
+        operation = AlterStorageEngine("mymodel", to_engine="InnoDB")
+        self.assertFalse(operation.reversible)
+
+        with self.assertRaises(NotImplementedError) as cm:
+            operation.database_backwards(None, None, None, None)
+
+        self.assertEqual(
+            str(cm.exception),
+            "You cannot reverse this operation"
+        )
+
+    def test_from_means_reversible(self):
+        operation = AlterStorageEngine("mymodel", from_engine="MyISAM",
+                                       to_engine="InnoDB")
+        self.assertTrue(operation.reversible)
+
+    def test_describe_without_from(self):
+        operation = AlterStorageEngine("Pony", "InnoDB")
+        self.assertEqual(operation.describe(),
+                         "Alter storage engine for Pony to InnoDB")
+
+    def test_describe_with_from(self):
+        operation = AlterStorageEngine("Pony", from_engine="MyISAM",
+                                       to_engine="InnoDB")
+        self.assertEqual(operation.describe(),
+                         "Alter storage engine for Pony from MyISAM to InnoDB")
+
+    def test_references_model(self):
+        operation = AlterStorageEngine("Pony", "InnoDB")
+        self.assertTrue(operation.references_model("PONY"))
+        self.assertTrue(operation.references_model("Pony"))
+        self.assertTrue(operation.references_model("pony"))
+        self.assertFalse(operation.references_model("poony"))
+
+    @override_mysql_variables(default_storage_engine='MyISAM')
+    def test_running_with_changes(self):
+        project_state = self.set_up_test_model("test_arstd")
+        operation = AlterStorageEngine("Pony", from_engine="MyISAM",
+                                       to_engine="InnoDB")
+
+        self.assertTableStorageEngine("test_arstd_pony", "MyISAM")
+
+        # Forwards
+        new_state = project_state.clone()
+        operation.state_forwards("test_arstd", new_state)
+        with connection.schema_editor() as editor:
+            operation.database_forwards("test_arstd", editor, project_state,
+                                        new_state)
+        self.assertTableStorageEngine("test_arstd_pony", "InnoDB")
+
+        # Backwards
+        with connection.schema_editor() as editor:
+            operation.database_backwards("test_arstd", editor, new_state,
+                                         project_state)
+        self.assertTableStorageEngine("test_arstd_pony", "MyISAM")
+
+    @override_mysql_variables(default_storage_engine='InnoDB')
+    def test_running_without_changes(self):
+        project_state = self.set_up_test_model("test_arstd")
+        operation = AlterStorageEngine("Pony", from_engine="MyISAM",
+                                       to_engine="InnoDB")
+
+        self.assertTableStorageEngine("test_arstd_pony", "InnoDB")
+
+        # Forwards - shouldn't actually do an ALTER since it is already InnoDB
+        new_state = project_state.clone()
+        operation.state_forwards("test_arstd", new_state)
+        capturer = CaptureQueriesContext(connection)
+        with capturer, connection.schema_editor() as editor:
+            operation.database_forwards("test_arstd", editor, project_state,
+                                        new_state)
+        queries = [q['sql'] for q in capturer.captured_queries]
+        self.assertFalse(
+            any(q.startswith('ALTER TABLE ') for q in queries),
+            "One of the executed queries was an unexpected ALTER TABLE:\n{}"
+            .format("\n".join(queries))
+        )
+        self.assertTableStorageEngine("test_arstd_pony", "InnoDB")
+
+        # Backwards - will actually ALTER since it is going 'back' to MyISAM
+        with connection.schema_editor() as editor:
+            operation.database_backwards("test_arstd", editor, new_state,
+                                         project_state)
+        self.assertTableStorageEngine("test_arstd_pony", "MyISAM")
+
+    # Copied from django core migration tests
+
+    def set_up_test_model(
+            self, app_label, second_model=False, third_model=False,
+            related_model=False, mti_model=False, proxy_model=False,
+            unique_together=False, options=False, db_table=None,
+            index_together=False):
+        """
+        Creates a test model state and database table.
+        """
+        # Delete the tables if they already exist
+        table_names = [
+            # Start with ManyToMany tables
+            '_pony_stables', '_pony_vans',
+            # Then standard model tables
+            '_pony', '_stable', '_van',
+        ]
+        tables = [(app_label + table_name) for table_name in table_names]
+        with connection.cursor() as cursor:
+            table_names = connection.introspection.table_names(cursor)
+            connection.disable_constraint_checking()
+            sql_delete_table = connection.schema_editor().sql_delete_table
+            with transaction.atomic():
+                for table in tables:
+                    if table in table_names:
+                        cursor.execute(sql_delete_table % {
+                            "table": connection.ops.quote_name(table),
+                        })
+            connection.enable_constraint_checking()
+
+        # Make the "current" state
+        model_options = {
+            "swappable": "TEST_SWAP_MODEL",
+            "index_together": [["weight", "pink"]] if index_together else [],
+            "unique_together": [["pink", "weight"]] if unique_together else [],
+        }
+        if options:
+            model_options["permissions"] = [("can_groom", "Can groom")]
+        if db_table:
+            model_options["db_table"] = db_table
+        operations = [migrations.CreateModel(
+            "Pony",
+            [
+                ("id", models.AutoField(primary_key=True)),
+                ("pink", models.IntegerField(default=3)),
+                ("weight", models.FloatField()),
+            ],
+            options=model_options,
+        )]
+        if second_model:
+            operations.append(migrations.CreateModel(
+                "Stable",
+                [
+                    ("id", models.AutoField(primary_key=True)),
+                ]
+            ))
+        if third_model:
+            operations.append(migrations.CreateModel(
+                "Van",
+                [
+                    ("id", models.AutoField(primary_key=True)),
+                ]
+            ))
+        if related_model:
+            operations.append(migrations.CreateModel(
+                "Rider",
+                [
+                    ("id", models.AutoField(primary_key=True)),
+                    ("pony", models.ForeignKey("Pony")),
+                    ("friend", models.ForeignKey("self"))
+                ],
+            ))
+        if mti_model:
+            operations.append(migrations.CreateModel(
+                "ShetlandPony",
+                fields=[
+                    ('pony_ptr', models.OneToOneField(
+                        auto_created=True,
+                        primary_key=True,
+                        to_field='id',
+                        serialize=False,
+                        to='Pony',
+                    )),
+                    ("cuteness", models.IntegerField(default=1)),
+                ],
+                bases=['%s.Pony' % app_label],
+            ))
+        if proxy_model:
+            operations.append(migrations.CreateModel(
+                "ProxyPony",
+                fields=[],
+                options={"proxy": True},
+                bases=['%s.Pony' % app_label],
+            ))
+
+        return self.apply_operations(app_label, ProjectState(), operations)
+
+    def apply_operations(self, app_label, project_state, operations):
+        migration = migrations.Migration('name', app_label)
+        migration.operations = operations
+        with connection.schema_editor() as editor:
+            return migration.apply(project_state, editor)
+
+    # Check functions
+
+    def assertTableStorageEngine(self, table_name, engine_name):
+        self.assertEqual(self.storage_engine(table_name), engine_name,
+                         "Table {} does not use the {} storage engine!"
+                         .format(table_name, engine_name))
+
+    def storage_engine(self, table_name):
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """SELECT ENGINE FROM INFORMATION_SCHEMA.TABLES
+                   WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s""",
+                (table_name,)
+            )
+            return cursor.fetchone()[0]