Repository: allegro/ralph

Issue:
ralph impossible to install (errno: 150)
```
root@dcim:~$ ralph migrate
Operations to perform:
  Synchronize unmigrated apps: dc_view, ralph2_sync, mptt, messages, ralph_admin, table, humanize, import_export, taggit_serializer, django_rq, permissions, staticfiles, rest_framework, foundation
  Apply all migrations: security, deployment, taggit, assets, transitions, admin, reports, custom_fields, accounts, supports, licences, reversion, virtual, data_center, sitetree, back_office, data_importer, operations, contenttypes, sessions, authtoken, auth, dhcp, external_services, networks, dashboards, attachments, domains
Synchronizing apps without migrations:
  Creating tables...
    Running deferred SQL...
  Installing custom SQL...
Running migrations:
  Rendering model states... DONE
  Applying contenttypes.0001_initial... OK
  Applying contenttypes.0002_remove_content_type_name... OK
  Applying auth.0001_initial... OK
  Applying auth.0002_alter_permission_name_max_length... OK
  Applying auth.0003_alter_user_email_max_length... OK
  Applying auth.0004_alter_user_username_opts... OK
  Applying auth.0005_alter_user_last_login_null... OK
  Applying auth.0006_require_contenttypes_0002... OK
  Applying accounts.0001_initial... OK
  Applying accounts.0002_auto_20151204_0758... OK
  Applying accounts.0003_region_country... OK
  Applying accounts.0004_region_stocktaking_enabled... OK
  Applying admin.0001_initial... OK
  Applying data_importer.0001_initial... OK
  Applying data_importer.0002_auto_20151125_1354... OK
  Applying data_importer.0003_auto_20160624_1217... OK
  Applying data_importer.0004_auto_20160728_1046... OK
  Applying taggit.0001_initial... OK
  Applying taggit.0002_auto_20150616_2121... OK
  Applying assets.0001_initial... OK
  Applying assets.0002_auto_20151125_1354... OK
  Applying assets.0003_auto_20151126_2205... OK
  Applying assets.0004_auto_20151204_0758... OK
  Applying assets.0005_category_depreciation_rate... OK
  Applying assets.0006_category_show_buyout_date... OK
  Applying assets.0007_auto_20160122_1022... OK
  Applying assets.0008_auto_20160122_1429... OK
  Applying assets.0009_auto_20160307_1138... OK
  Applying assets.0010_auto_20160405_1531... OK
  Applying assets.0011_auto_20160603_0742... OK
  Applying assets.0012_auto_20160606_1409... OK
  Applying assets.0013_auto_20160615_2140... OK
  Applying assets.0014_memory... OK
  Applying assets.0015_auto_20160701_0952... OK
  Applying assets.0016_fibrechannelcard... OK
  Applying assets.0017_processor... OK
  Applying assets.0018_disk... OK
  Applying assets.0019_auto_20160719_1443... OK
  Applying assets.0020_auto_20160803_0712...Traceback (most recent call last):
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/mysql/base.py", line 124, in execute
    return self.cursor.execute(query, args)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 220, in execute
    self.errorhandler(self, exc, value)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
    raise errorvalue
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 209, in execute
    r = self._query(query)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 371, in _query
    rowcount = self._do_query(q)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 335, in _do_query
    db.query(q)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/connections.py", line 280, in query
    _mysql.connection.query(self, query)
_mysql_exceptions.OperationalError: (1025, "Error on rename of './ralph/#sql-3c8_98' to './ralph/assets_configurationclass' (errno: 150)")

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/opt/ralph/ralph-core/bin/ralph", line 9, in <module>
    load_entry_point('ralph==3.0.0', 'console_scripts', 'ralph')()
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/ralph/__main__.py", line 29, in prod
    main('ralph.settings.prod')
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/ralph/__main__.py", line 14, in main
    execute_from_command_line(sys.argv)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/core/management/__init__.py", line 354, in execute_from_command_line
    utility.execute()
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/core/management/__init__.py", line 346, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/core/management/base.py", line 394, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/core/management/base.py", line 445, in execute
    output = self.handle(*args, **options)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/core/management/commands/migrate.py", line 222, in handle
    executor.migrate(targets, plan, fake=fake, fake_initial=fake_initial)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/migrations/executor.py", line 110, in migrate
    self.apply_migration(states[migration], migration, fake=fake, fake_initial=fake_initial)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/migrations/executor.py", line 148, in apply_migration
    state = migration.apply(state, schema_editor)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/migrations/migration.py", line 115, in apply
    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/migrations/operations/special.py", line 43, in database_forwards
    database_operation.database_forwards(app_label, schema_editor, from_state, to_state)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/migrations/operations/fields.py", line 275, in database_forwards
    to_model._meta.get_field(self.new_name),
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/base/schema.py", line 484, in alter_field
    old_db_params, new_db_params, strict)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/base/schema.py", line 547, in _alter_field
    self.execute(self._rename_field_sql(model._meta.db_table, old_field, new_field, new_type))
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/base/schema.py", line 111, in execute
    cursor.execute(sql, params)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/utils.py", line 98, in __exit__
    six.reraise(dj_exc_type, dj_exc_value, traceback)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/utils/six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/utils.py", line 64, in execute
    return self.cursor.execute(sql, params)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/django/db/backends/mysql/base.py", line 124, in execute
    return self.cursor.execute(query, args)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 220, in execute
    self.errorhandler(self, exc, value)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
    raise errorvalue
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 209, in execute
    r = self._query(query)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 371, in _query
    rowcount = self._do_query(q)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/cursors.py", line 335, in _do_query
    db.query(q)
  File "/opt/ralph/ralph-core/lib/python3.4/site-packages/MySQLdb/connections.py", line 280, in query
    _mysql.connection.query(self, query)
django.db.utils.OperationalError: (1025, "Error on rename of './ralph/#sql-3c8_98' to './ralph/assets_configurationclass' (errno: 150)")
```



Discussion:
version is latest... i try another version 7080, 7076 and 7074... same problem too

I got the same problem too.
And downgrade to ralph-core_3.0.0-snapshot-20160805-6680, then works well.
It seems that the latest version have problems.

@gerickt @yiyuandao please refer to https://github.com/allegro/ralph/issues/2769 - we'll continue to fix this bug there.


Resolving patch:
diff --git a/src/ralph/assets/_migration_helpers.py b/src/ralph/assets/_migration_helpers.py
--- a/src/ralph/assets/_migration_helpers.py
+++ b/src/ralph/assets/_migration_helpers.py
@@ -3,9 +3,11 @@
 
 import logging
 from collections import defaultdict
+from contextlib import ContextDecorator
 from functools import partial
 
 from django.db import connection, migrations, models
+from django.db.backends.base.schema import _related_non_m2m_objects
 
 logger = logging.getLogger(__name__)
 
@@ -111,6 +113,89 @@ def _foreign_key_check_wrap(operations):
     return operations
 
 
+class DropAndCreateForeignKey(ContextDecorator):
+    """
+    Drop FK to old_field before migration operation and recreate FK (to the new
+    field) after migration operation.
+    """
+    def __init__(self, old_field, new_field, schema_editor):
+        self.old_field = old_field
+        self.new_field = new_field
+        self.schema_editor = schema_editor
+
+    def __enter__(self):
+        # copy from django.db.backends.base.schema.BaseDatabaseSchemaEditor._alter_field  # noqa
+        # drop any FK pointing to old_field
+        for _old_rel, new_rel in _related_non_m2m_objects(
+            self.old_field, self.new_field
+        ):
+            rel_fk_names = self.schema_editor._constraint_names(
+                new_rel.related_model, [new_rel.field.column], foreign_key=True
+            )
+            for fk_name in rel_fk_names:
+                logger.debug('Dropping FK to {} ({})'.format(
+                    new_rel.field, fk_name
+                ))
+                self.schema_editor.execute(
+                    self.schema_editor._delete_constraint_sql(
+                        self.schema_editor.sql_delete_fk,
+                        new_rel.related_model,
+                        fk_name
+                    )
+                )
+
+    def __exit__(self, *exc):
+        # copy from django.db.backends.base.schema.BaseDatabaseSchemaEditor._alter_field  # noqa
+        # (re)create any FK pointing to the new field
+        for rel in self.new_field.model._meta.related_objects:
+            if not rel.many_to_many:
+                logger.debug('Recreating FK to {}'.format(rel.field))
+                self.schema_editor.execute(
+                    self.schema_editor._create_fk_sql(
+                        rel.related_model, rel.field, "_fk"
+                    )
+                )
+
+
+class RenameFieldWithFKDrop(migrations.RenameField):
+    """
+    Rename field with dropping any FK pointing to it before actual renaming
+    and recreating them after renaming.
+
+    It's especially usefull for MySQL 5.5, where renaming field that is
+    referenced by another table is not supported (errno 150). The workaround
+    here is to drop FK first, then rename field, and then recreate FK. It's
+    supported fully since MySQL 5.6.6.
+
+    Resources:
+    https://code.djangoproject.com/ticket/24995
+    https://github.com/django/django/pull/4881
+    http://dev.mysql.com/doc/refman/5.5/en/create-table-foreign-keys.html
+    http://stackoverflow.com/questions/2014498/renaming-foreign-key-columns-in-mysql  # noqa
+    """
+    def database_forwards(
+        self, app_label, schema_editor, from_state, to_state
+    ):
+        to_model = to_state.apps.get_model(app_label, self.model_name)
+        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
+            from_model = from_state.apps.get_model(app_label, self.model_name)
+            old_field = from_model._meta.get_field(self.old_name)
+            new_field = to_model._meta.get_field(self.new_name)
+            with DropAndCreateForeignKey(old_field, new_field, schema_editor):
+                schema_editor.alter_field(from_model, old_field, new_field)
+
+    def database_backwards(
+        self, app_label, schema_editor, from_state, to_state
+    ):
+        to_model = to_state.apps.get_model(app_label, self.model_name)
+        if self.allow_migrate_model(schema_editor.connection.alias, to_model):
+            from_model = from_state.apps.get_model(app_label, self.model_name)
+            old_field = from_model._meta.get_field(self.new_name)
+            new_field = to_model._meta.get_field(self.old_name)
+            with DropAndCreateForeignKey(old_field, new_field, schema_editor):
+                schema_editor.alter_field(from_model, old_field, new_field)
+
+
 class InheritFromBaseObject(migrations.SeparateDatabaseAndState):
     """
     Handle migrating model to inherit from BaseObject.
@@ -148,7 +233,7 @@ def __init__(self, app_name, model_name, rewrite_fields=None):
         ]
 
         database_operations = [
-            migrations.RenameField(
+            RenameFieldWithFKDrop(
                 model_name=model_name.lower(),
                 old_name='id',
                 new_name='baseobject_ptr_id'


Test patch:
diff --git a/src/ralph/security/tests/factories.py b/src/ralph/security/tests/factories.py
--- a/src/ralph/security/tests/factories.py
+++ b/src/ralph/security/tests/factories.py
@@ -39,7 +39,7 @@ class VulnerabilityFactory(DjangoModelFactory):
 
     name = factory.Sequence(lambda n: 'vulnserability %d' % n)
     patch_deadline = factory.LazyAttribute(
-        lambda o: datetime.now() + timedelta(days=10)
+        lambda o: (datetime.now() + timedelta(days=10)).replace(microsecond=0)
     )
     risk = Risk.low
     external_vulnerability_id = factory.Sequence(lambda n: n)
diff --git a/src/ralph/security/tests/test_api.py b/src/ralph/security/tests/test_api.py
--- a/src/ralph/security/tests/test_api.py
+++ b/src/ralph/security/tests/test_api.py
@@ -94,11 +94,11 @@ def test_patch_security_scan(self):
         data = {
             'last_scan_date': (
                 datetime.now() + timedelta(days=10)
-            ).isoformat(),
+            ).replace(microsecond=0).isoformat(),
             'scan_status': ScanStatus.error.name,
             'next_scan_date': (
                 datetime.now() + timedelta(days=15)
-            ).isoformat(),
+            ).replace(microsecond=0).isoformat(),
             'details_url': self.security_scan.details_url + '-new',
             'rescan_url': self.security_scan.rescan_url + '-new',
             'host_ip': ip.address,
@@ -148,7 +148,9 @@ def test_get_vulnerability_details(self):
         self.assertEqual(response.data['name'], self.vulnerability.name)
         self.assertEqual(
             response.data['patch_deadline'],
-            self.vulnerability.patch_deadline.isoformat(),
+            self.vulnerability.patch_deadline.replace(
+                microsecond=0
+            ).isoformat(),
         )
         self.assertEqual(response.data['risk'], Risk.low.name)
         self.assertEqual(
@@ -162,7 +164,7 @@ def test_create_vulnerability(self):
             'name': "vulnerability name",
             'patch_deadline': (
                 datetime.now() + timedelta(days=10)
-            ).isoformat(),
+            ).replace(microsecond=0).isoformat(),
             'risk': Risk.low.name,
             'external_vulnerability_id': 100,
         }
@@ -186,7 +188,7 @@ def test_patch_vulnerability(self):
             'name': self.vulnerability.name + ' new',
             'patch_deadline': (
                 self.vulnerability.patch_deadline + timedelta(days=3)
-            ).isoformat(),
+            ).replace(microsecond=0).isoformat(),
             'risk': Risk.high.name,
             'external_vulnerability_id': self.vulnerability.external_vulnerability_id + 10  # noqa
         }