Repository: fedora-infra/bodhi

Issue:
The new batched migration fails to run under BDR
Bodhi's [auto-DDL BDR locking](https://github.com/fedora-infra/bodhi/blob/a5d6b4eac337fd5288f3b96b5446b4704e518384/alembic/env.py#L84) doesn't seem to work with a [recent migration to add a "batched" state](https://github.com/fedora-infra/bodhi/blob/a5d6b4eac337fd5288f3b96b5446b4704e518384/alembic/versions/2a10629168e4_add_request_for_batched_update.py) to the Update.request enum:

```
INFO  [alembic.env] Emitting SQL to allow for global DDL locking with BDR
INFO  [alembic.runtime.migration] Running upgrade 0e105f6e36b4 -> 2a10629168e4, Add request for batched update
Traceback (most recent call last):
  File "/usr/bin/alembic", line 12, in <module>
    sys.exit(load_entry_point('alembic', 'console_scripts', 'alembic')())
  File "/usr/lib/python2.7/site-packages/alembic/config.py", line 479, in main
    CommandLine(prog=prog).main(argv=argv)
  File "/usr/lib/python2.7/site-packages/alembic/config.py", line 473, in main
    self.run_cmd(cfg, options)
  File "/usr/lib/python2.7/site-packages/alembic/config.py", line 456, in run_cmd
    **dict((k, getattr(options, k)) for k in kwarg)
  File "/usr/lib/python2.7/site-packages/alembic/command.py", line 174, in upgrade
    script.run_env()
  File "/usr/lib/python2.7/site-packages/alembic/script/base.py", line 397, in run_env
    util.load_python_file(self.dir, 'env.py')
  File "/usr/lib/python2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
    module = load_module_py(module_id, path)
  File "/usr/lib/python2.7/site-packages/alembic/util/compat.py", line 79, in load_module_py
    mod = imp.load_source(module_id, path, fp)
  File "/usr/share/bodhi/alembic/env.py", line 94, in <module>
    run_migrations_online()
  File "/usr/share/bodhi/alembic/env.py", line 86, in run_migrations_online
    context.run_migrations()
  File "<string>", line 8, in run_migrations
  File "/usr/lib/python2.7/site-packages/alembic/runtime/environment.py", line 797, in run_migrations
    self.get_context().run_migrations(**kw)
  File "/usr/lib/python2.7/site-packages/alembic/runtime/migration.py", line 312, in run_migrations
    step.migration_fn(**kw)
  File "/usr/share/bodhi/alembic/versions/2a10629168e4_add_request_for_batched_update.py", line 18, in upgrade
    op.execute("ALTER TYPE ck_update_request ADD VALUE 'batched' AFTER 'stable'")
  File "<string>", line 8, in execute
  File "<string>", line 3, in execute
  File "/usr/lib/python2.7/site-packages/alembic/operations/ops.py", line 1826, in execute
    return operations.invoke(op)
  File "/usr/lib/python2.7/site-packages/alembic/operations/base.py", line 318, in invoke
    return fn(self, operation)
  File "/usr/lib/python2.7/site-packages/alembic/operations/toimpl.py", line 161, in execute_sql
    execution_options=operation.execution_options
  File "/usr/lib/python2.7/site-packages/alembic/ddl/impl.py", line 121, in execute
    self._exec(sql, execution_options)
  File "/usr/lib/python2.7/site-packages/alembic/ddl/impl.py", line 118, in _exec
    return conn.execute(construct, *multiparams, **params)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 914, in execute
    return meth(self, multiparams, params)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
    compiled_sql, distilled_params
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
    context)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1341, in _handle_dbapi_exception
    exc_info
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
    context)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/default.py", line 450, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) Global DDL locking attempt rejected by configuration
DETAIL:  bdr.permit_ddl_locking is false and the attempted command would require the global lock to be acquired. Command rejected.
HINT:  See the 'DDL replication' chapter of the documentation.
 [SQL: "ALTER TYPE ck_update_request ADD VALUE 'batched' AFTER 'stable'"]
```


Discussion:
My guess is that we might need to manually ask for the DDL lock again in that migration, since it runs a ```COMMIT``` before adding the enum.
I have this patch on staging right now:

```
diff --git a/alembic/versions/2a10629168e4_add_request_for_batched_update.py b/alembic/versions/2a10629168e4_add_request_for_batched_update.py
index 683037c..f575d21 100644
--- a/alembic/versions/2a10629168e4_add_request_for_batched_update.py
+++ b/alembic/versions/2a10629168e4_add_request_for_batched_update.py
@@ -5,6 +5,7 @@ Revises: 0e105f6e36b4
 Create Date: 2017-06-23 22:14:55.464629
 """
 from alembic import op
+from sqlalchemy import exc
 
 
 # revision identifiers, used by Alembic.
@@ -15,6 +16,17 @@ down_revision = '0e105f6e36b4'
 def upgrade():
     """ Add a 'batched' request enum value in ck_update_request. """
     op.execute('COMMIT')  # See https://bitbucket.org/zzzeek/alembic/issue/123
+    try:
+        # This will raise a ProgrammingError if the DB server doesn't use BDR.
+        op.execute('SHOW bdr.permit_ddl_locking')
+        # This server uses BDR, so let's ask for a DDL lock.
+        op.execute('BEGIN')
+        op.execute('SET LOCAL bdr.permit_ddl_locking = true')
+        print 'DDL acquired.'
+    except exc.ProgrammingError:
+        # This server doesn't use BDR, so no problem.
+        print 'DDL not acquired.'
+        pass
     op.execute("ALTER TYPE ck_update_request ADD VALUE 'batched' AFTER 'stable'")
```

However, staging seems unable to get the DDL lock again:

```
[root@bodhi-backend01 ~][STG]# alembic -c /etc/bodhi/alembic.ini upgrade head
INFO  [alembic.runtime.migration] Context impl PostgresqlImpl.
INFO  [alembic.runtime.migration] Will assume transactional DDL.
postgres_bdr: True
INFO  [alembic.env] Emitting SQL to allow for global DDL locking with BDR
INFO  [alembic.runtime.migration] Running upgrade 0e105f6e36b4 -> 2a10629168e4, Add request for batched update
DDL acquired.
Traceback (most recent call last):
  File "/usr/bin/alembic", line 12, in <module>
    sys.exit(load_entry_point('alembic', 'console_scripts', 'alembic')())
  File "/usr/lib/python2.7/site-packages/alembic/config.py", line 479, in main
    CommandLine(prog=prog).main(argv=argv)
  File "/usr/lib/python2.7/site-packages/alembic/config.py", line 473, in main
    self.run_cmd(cfg, options)
  File "/usr/lib/python2.7/site-packages/alembic/config.py", line 456, in run_cmd
    **dict((k, getattr(options, k)) for k in kwarg)
  File "/usr/lib/python2.7/site-packages/alembic/command.py", line 174, in upgrade
    script.run_env()
  File "/usr/lib/python2.7/site-packages/alembic/script/base.py", line 397, in run_env
    util.load_python_file(self.dir, 'env.py')
  File "/usr/lib/python2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
    module = load_module_py(module_id, path)
  File "/usr/lib/python2.7/site-packages/alembic/util/compat.py", line 79, in load_module_py
    mod = imp.load_source(module_id, path, fp)
  File "/usr/share/bodhi/alembic/env.py", line 94, in <module>
    run_migrations_online()
  File "/usr/share/bodhi/alembic/env.py", line 86, in run_migrations_online
    context.run_migrations()
  File "<string>", line 8, in run_migrations
  File "/usr/lib/python2.7/site-packages/alembic/runtime/environment.py", line 797, in run_migrations
    self.get_context().run_migrations(**kw)
  File "/usr/lib/python2.7/site-packages/alembic/runtime/migration.py", line 312, in run_migrations
    step.migration_fn(**kw)
  File "/usr/share/bodhi/alembic/versions/2a10629168e4_add_request_for_batched_update.py", line 30, in upgrade
    op.execute("ALTER TYPE ck_update_request ADD VALUE 'batched' AFTER 'stable'")
  File "<string>", line 8, in execute
  File "<string>", line 3, in execute
  File "/usr/lib/python2.7/site-packages/alembic/operations/ops.py", line 1826, in execute
    return operations.invoke(op)
  File "/usr/lib/python2.7/site-packages/alembic/operations/base.py", line 318, in invoke
    return fn(self, operation)
  File "/usr/lib/python2.7/site-packages/alembic/operations/toimpl.py", line 161, in execute_sql
    execution_options=operation.execution_options
  File "/usr/lib/python2.7/site-packages/alembic/ddl/impl.py", line 121, in execute
    self._exec(sql, execution_options)
  File "/usr/lib/python2.7/site-packages/alembic/ddl/impl.py", line 118, in _exec
    return conn.execute(construct, *multiparams, **params)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 914, in execute
    return meth(self, multiparams, params)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
    compiled_sql, distilled_params
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
    context)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1341, in _handle_dbapi_exception
    exc_info
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
    context)
  File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/default.py", line 450, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) No peer nodes or peer node count unknown, cannot acquire global lock
HINT:  BDR is probably still starting up, wait a while
 [SQL: "ALTER TYPE ck_update_request ADD VALUE 'batched' AFTER 'stable'"]
```

Resolving patch:
diff --git a/alembic/versions/2a10629168e4_add_request_for_batched_update.py b/alembic/versions/2a10629168e4_add_request_for_batched_update.py
--- a/alembic/versions/2a10629168e4_add_request_for_batched_update.py
+++ b/alembic/versions/2a10629168e4_add_request_for_batched_update.py
@@ -5,6 +5,7 @@
 Create Date: 2017-06-23 22:14:55.464629
 """
 from alembic import op
+from sqlalchemy import exc
 
 
 # revision identifiers, used by Alembic.
@@ -15,6 +16,14 @@
 def upgrade():
     """ Add a 'batched' request enum value in ck_update_request. """
     op.execute('COMMIT')  # See https://bitbucket.org/zzzeek/alembic/issue/123
+    try:
+        # This will raise a ProgrammingError if the DB server doesn't use BDR.
+        op.execute('SHOW bdr.permit_ddl_locking')
+        # This server uses BDR, so let's ask for a DDL lock.
+        op.execute('SET LOCAL bdr.permit_ddl_locking = true')
+    except exc.ProgrammingError:
+        # This server doesn't use BDR, so no problem.
+        pass
     op.execute("ALTER TYPE ck_update_request ADD VALUE 'batched' AFTER 'stable'")
 
 


Test patch: