Repository: meltano/meltano

Issue:
bug: alembic downgrade from 6828cc5b1a4f to 5b43800443d1 fails.
### Meltano Version

downgrading from 2+ to 1.76

### Python Version

3.7

### Bug scope

Other

### Operating System

Windows, but this probably isn't OS specific.

### Description

Relates to #6870. Ran `meltano repl` (as advised by @DouweM) then:

```python
from alembic import command
from alembic.config import Config
from meltano.migrations import MIGRATION_DIR
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory

engine, session = project_engine(project, default=True)
conn = engine.connect()
cfg = Config()
cfg.attributes["connection"] = conn
cfg.set_main_option("script_location", str(MIGRATION_DIR))
script = ScriptDirectory.from_config(cfg)
context = MigrationContext.configure(conn)
command.downgrade(cfg, "d135f52a6f49")
```

Tracelog:

```
2022-10-11T11:24:19.745585Z [info     ] Context impl PostgresqlImpl.
2022-10-11T11:24:19.747541Z [info     ] Will assume transactional DDL.
2022-10-11T11:24:19.853452Z [info     ] Running downgrade 6828cc5b1a4f -> f4c225a9492f, Create dedicated state table
2022-10-11T11:24:19.898474Z [info     ] Running downgrade f4c225a9492f -> 5b43800443d1, Create dedicated job_state table ---------------------------------------------------------------------------
UndefinedTable                            Traceback (most recent call last)
~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\base.py in _execute_context(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)
   1900                     self.dialect.do_execute(
-> 1901                         cursor, statement, parameters, context
   1902                     )

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\default.py in do_execute(self, cursor, statement, parameters, context)
    735     def do_execute(self, cursor, statement, parameters, context=None):
--> 736         cursor.execute(statement, parameters)
    737

UndefinedTable: table "state" does not exist


The above exception was the direct cause of the following exception:

ProgrammingError                          Traceback (most recent call last)
<ipython-input-14-8baf3a7dd97e> in <module>
----> 1 command.downgrade(cfg, "d135f52a6f49")

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\command.py in downgrade(config, revision, sql, tag)
    366         tag=tag,
    367     ):
--> 368         script.run_env()
    369
    370

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\script\base.py in run_env(self)
    567
    568         """
--> 569         util.load_python_file(self.dir, "env.py")
    570
    571     @property

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\util\pyfiles.py in load_python_file(dir_, filename)
     92     if ext == ".py":
     93         if os.path.exists(path):
---> 94             module = load_module_py(module_id, path)
     95         else:
     96             pyc_path = pyc_file_from_path(path)

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\util\pyfiles.py in load_module_py(module_id, path)
    108     assert spec
    109     module = importlib.util.module_from_spec(spec)
--> 110     spec.loader.exec_module(module)  # type: ignore
    111     return module

~\AppData\Local\Programs\Python\Python37\lib\importlib\_bootstrap_external.py in exec_module(self, module)

~\AppData\Local\Programs\Python\Python37\lib\importlib\_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\meltano\migrations\env.py in <module>
     58     run_migrations_offline()
     59 else:
---> 60     run_migrations_online()

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\meltano\migrations\env.py in run_migrations_online()
     35     connection = config.attributes.get("connection")
     36     if connection:
---> 37         _run_migrations_online(connection)
     38         return
     39

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\meltano\migrations\env.py in _run_migrations_online(connection)
     52
     53     with context.begin_transaction():
---> 54         context.run_migrations()
     55
     56

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\context.py in run_migrations(*args, **kw)

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\runtime\environment.py in run_migrations(self, **kw)
    851         assert self._migration_context is not None
    852         with Operations.context(self._migration_context):
--> 853             self.get_context().run_migrations(**kw)
    854
    855     def execute(self, sql, execution_options=None):

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\runtime\migration.py in run_migrations(self, **kw)
    621                         "-- Running %s" % (step.short_log,)
    622                     )
--> 623                 step.migration_fn(**kw)
    624
    625                 # previously, we wouldn't stamp per migration

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\meltano\migrations\versions\f4c225a9492f_create_dedicated_job_state_table.py in downgrade()
     25     # Remove job_state table
     26     # Job run history is still maintained, so no need to copy
---> 27     op.drop_table("state")

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\op.py in drop_table(*args, **kw)

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\operations\base.py in drop_table(self, table_name, schema, **kw)

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\operations\ops.py in drop_table(cls, operations, table_name, schema, **kw)
   1347         """
   1348         op = cls(table_name, schema=schema, table_kw=kw)
-> 1349         operations.invoke(op)
   1350
   1351

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\operations\base.py in invoke(self, operation)
    397             operation, self.migration_context.impl.__dialect__
    398         )
--> 399         return fn(self, operation)
    400
    401     def f(self, name: str) -> "conv":

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\operations\toimpl.py in drop_table(operations, operation)
     78 def drop_table(operations: "Operations", operation: "ops.DropTableOp") -> None:
     79     operations.impl.drop_table(
---> 80         operation.to_table(operations.migration_context)
     81     )
     82

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\ddl\impl.py in drop_table(self, table)
    375             table, self.connection, checkfirst=False, _ddl_runner=self
    376         )
--> 377         self._exec(schema.DropTable(table))
    378         table.dispatch.after_drop(
    379             table, self.connection, checkfirst=False, _ddl_runner=self

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\alembic\ddl\impl.py in _exec(self, construct, execution_options, multiparams, params)
    193                 multiparams += (params,)
    194
--> 195             return conn.execute(construct, multiparams)
    196
    197     def execute(

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\base.py in execute(self, statement, *multiparams, **params)
   1378             )
   1379         else:
-> 1380             return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
   1381
   1382     def _execute_function(self, func, multiparams, params, execution_options):

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\sql\ddl.py in _execute_on_connection(self, connection, multiparams, params, execution_options)
     79     ):
     80         return connection._execute_ddl(
---> 81             self, multiparams, params, execution_options
     82         )
     83

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\base.py in _execute_ddl(self, ddl, multiparams, params, execution_options)
   1476             None,
   1477             execution_options,
-> 1478             compiled,
   1479         )
   1480         if self._has_events or self.engine._has_events:

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\base.py in _execute_context(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)
   1942         except BaseException as e:
   1943             self._handle_dbapi_exception(
-> 1944                 e, statement, parameters, cursor, context
   1945             )
   1946

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   2123             elif should_wrap:
   2124                 util.raise_(
-> 2125                     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2126                 )
   2127             else:

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\util\compat.py in raise_(***failed resolving arguments***)
    206
    207         try:
--> 208             raise exception
    209         finally:
    210             # credit to

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\base.py in _execute_context(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)
   1899                 if not evt_handled:
   1900                     self.dialect.do_execute(
-> 1901                         cursor, statement, parameters, context
   1902                     )
   1903

~\.virtualenvs\meltano-9aGVMBGe\lib\site-packages\sqlalchemy\engine\default.py in do_execute(self, cursor, statement, parameters, context)
    734
    735     def do_execute(self, cursor, statement, parameters, context=None):
--> 736         cursor.execute(statement, parameters)
    737
    738     def do_execute_no_params(self, cursor, statement, context=None):

ProgrammingError: (psycopg2.errors.UndefinedTable) table "state" does not exist
```

# Diagnosis
- Downgrade path in first migration drops table `state`: https://github.com/meltano/meltano/blob/v2.7.2/src/meltano/migrations/versions/6828cc5b1a4f_create_dedicated_state_table.py#L367
- Downgrade path in second migration also drops table `state`: https://github.com/meltano/meltano/blob/v2.7.2/src/meltano/migrations/versions/f4c225a9492f_create_dedicated_job_state_table.py#L27

The second step fails because `state` is already dropped.

# Suggested Fix

Remove the `f4c225a9492f` migration. It does nothing during upgrade, and only causes problems on downgrade.

### Code

_No response_


Discussion:
@cjohnhanson Can you please look at this? I know you've been looking into these migrations recently.
As a slight update/workaround. Forcibly skipping this migration resolves the issue but I think it's a more technical hack than we should expect from most users.

Running this workaround after getting far enough to experience this issue jumps over that issue:
```python
conn.execute("update alembic_version set version_num='5b43800443d1'")
```

All the migrations beyond this (as least as far as `d135f52a6f49`) were not problematic.


Resolving patch:
diff --git a/src/meltano/migrations/versions/f4c225a9492f_create_dedicated_job_state_table.py b/src/meltano/migrations/versions/f4c225a9492f_create_dedicated_job_state_table.py
--- a/src/meltano/migrations/versions/f4c225a9492f_create_dedicated_job_state_table.py
+++ b/src/meltano/migrations/versions/f4c225a9492f_create_dedicated_job_state_table.py
@@ -1,4 +1,7 @@
-"""Create dedicated job_state table
+"""THIS IS A NO-OP.
+
+IT EXISTS ONLY TO PREVENT USERS WHO APPLIED THIS MIGRATION
+FROM ENDING UP WITH AN ORPHANED alembic_version IN THEIR DB.
 
 Revision ID: f4c225a9492f
 Revises: 5b43800443d1
@@ -17,11 +20,8 @@
 
 
 def upgrade():
-    # Create state table
     pass
 
 
 def downgrade():
-    # Remove job_state table
-    # Job run history is still maintained, so no need to copy
-    op.drop_table("state")
+    pass


Test patch: