# Buggy code
def thd(engine):
    # if the migrate_version table exists, we can just let migrate
    # take care of this process.
    if table_exists(engine, "migrate_version"):
        r = engine.execute("select version from migrate_version limit 1")
        old_version = r.scalar()
        if old_version < 40:
            raise EightUpgradeError()
        upgrade(engine)

    # if the version table exists, then we can version_control things
    # at that version, drop the version table, and let migrate take
    # care of the rest.
    elif table_exists(engine, "version"):
        raise EightUpgradeError()

    # otherwise, this db is new, so we don't bother using the migration engine
    # and just create the tables, and put the version directly to
    # latest
    else:
        # do some tests before getting started
        test_unicode(engine)

        log.msg("Initializing empty database")
        Model.metadata.create_all(engine)
        repo = migrate.versioning.repository.Repository(self.repo_path)

        version_control(engine, repo.latest)


# Traceback (sqlalchemy.exc.NoSuchTableError)
$doas -u _buildbot buildbot upgrade-master /build/buildbot9
checking basedir
checking for running master
checking master.cfg
upgrading basedir
upgrading database (sqlite:/state.sqlite)
Warning: Stopping this process might cause data loss
Got fatal Exception on DB
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/twisted/python/threadpool.py", line 266, in <lambda>
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/twisted/python/context.py", line 122, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/twisted/python/context.py", line 85, in callWithContext
return func(*args,**kw)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/pool.py", line 209, in __thd
log.err(e, 'Got fatal Exception on DB')
--- <exception caught here> ---
File "/usr/local/lib/python3.7/site-packages/buildbot/db/pool.py", line 182, in __thd
rv = callable(arg, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/model.py", line 903, in thd
upgrade(engine)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/model.py", line 868, in upgrade
schema.runchange(version, change, 1)
File "/usr/local/lib/python3.7/site-packages/migrate/versioning/schema.py", line 93, in runchange
change.run(self.engine, step)
File "/usr/local/lib/python3.7/site-packages/migrate/versioning/script/py.py", line 148, in run
script_func(engine)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/migrate/versions/054_add_index_to_steps.py", line 26, in upgrade
steps = sautils.Table('steps', metadata, autoload=True)
File "/usr/local/lib/python3.7/site-packages/buildbot/util/sautils.py", line 62, in Table
return sa.Table(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 559, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2056, in run_callable
return conn.run_callable(callable_, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 626, in reflecttable
raise exc.NoSuchTableError(table.name)
sqlalchemy.exc.NoSuchTableError: migration_tmp

problem while upgrading!:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/buildbot/scripts/upgrade_master.py", line 141, in _upgradeMaster
yield upgradeDatabase(config, master_cfg)
File "/usr/local/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/local/lib/python3.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/local/lib/python3.7/site-packages/buildbot/scripts/upgrade_master.py", line 103, in upgradeDatabase
yield db.model.upgrade()
File "/usr/local/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/local/lib/python3.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/model.py", line 925, in upgrade
yield self.db.pool.do_with_engine(thd)
File "/usr/local/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/local/lib/python3.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/pool.py", line 228, in do_with_engine
args, kwargs)
File "/usr/local/lib/python3.7/site-packages/twisted/python/threadpool.py", line 250, in inContext
result = inContext.theWork()
File "/usr/local/lib/python3.7/site-packages/twisted/python/threadpool.py", line 266, in <lambda>
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/twisted/python/context.py", line 122, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/twisted/python/context.py", line 85, in callWithContext
return func(*args,**kw)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/pool.py", line 182, in __thd
rv = callable(arg, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/model.py", line 903, in thd
upgrade(engine)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/model.py", line 868, in upgrade
schema.runchange(version, change, 1)
File "/usr/local/lib/python3.7/site-packages/migrate/versioning/schema.py", line 93, in runchange
change.run(self.engine, step)
File "/usr/local/lib/python3.7/site-packages/migrate/versioning/script/py.py", line 148, in run
script_func(engine)
File "/usr/local/lib/python3.7/site-packages/buildbot/db/migrate/versions/054_add_index_to_steps.py", line 26, in upgrade
steps = sautils.Table('steps', metadata, autoload=True)
File "/usr/local/lib/python3.7/site-packages/buildbot/util/sautils.py", line 62, in Table
return sa.Table(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 559, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2056, in run_callable
return conn.run_callable(callable_, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 633, in reflecttable
exclude_columns, _extend_on, reflection_options)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 761, in _reflect_fk
**reflection_options
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 456, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 451, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 533, in _init
include_columns, _extend_on=_extend_on)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 546, in _autoload
_extend_on=_extend_on
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1545, in run_callable
return callable_(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 390, in reflecttable
table, include_columns, exclude_columns, **opts)
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/reflection.py", line 626, in reflecttable
raise exc.NoSuchTableError(table.name)
sqlalchemy.exc.NoSuchTableError: migration_tmp

# Fixed code
def thd(engine):
    # if the migrate_version table exists, we can just let migrate
    # take care of this process.
    if table_exists(engine, "migrate_version"):
        r = engine.execute("select version from migrate_version limit 1")
        old_version = r.scalar()
        if old_version < 40:
            raise EightUpgradeError()
        try:
            upgrade(engine)
        except sa.exc.NoSuchTableError as e:  # pragma: no cover
            if "migration_tmp" in str(e):
                log.err(
                    "A serious error has been encountered during the upgrade. The "
                    "previous upgrade has been likely interrupted. The database has "
                    "been damaged and automatic recovery is impossible."
                )
                log.err(
                    "If you believe this is an error, please submit a bug to the "
                    "Buildbot project."
                )
            raise

    # if the version table exists, then we can version_control things
    # at that version, drop the version table, and let migrate take
    # care of the rest.
    elif table_exists(engine, "version"):
        raise EightUpgradeError()

    # otherwise, this db is new, so we don't bother using the migration engine
    # and just create the tables, and put the version directly to
    # latest
    else:
        # do some tests before getting started
        test_unicode(engine)

        log.msg("Initializing empty database")
        Model.metadata.create_all(engine)
        repo = migrate.versioning.repository.Repository(self.repo_path)

        version_control(engine, repo.latest)