Repository: rucio/rucio

Issue:
tools/reset_database.py should remove postgres types
Motivation
----------
Since the sqlalchemy update (#4133) I am getting errors that types already exist on postgres after resetting the database (with or without --drop-everything) in my docker-compose environment.


Modification
------------
Since the `tools/reset_database.py` script is meant to "destroy" the database and leave nothing behind, one option would be to remove all types found in the database.


Discussion:
Correct, this is because the types always end up in the `public` schema of PostgreSQL.

Quickest way to do this, in two steps:

`drop schema if exists public cascade;`
`create schema public;`
That seems to help a bit, but some types seem to be in the dev schema and aren't removed either:

```
sqlalchemy.exc.ProgrammingError: (psycopg2.errors.DuplicateObject) type "RULES_HIST_RECENT_DIDTYPE_CHK" already exists

[SQL: CREATE TYPE dev."RULES_HIST_RECENT_DIDTYPE_CHK" AS ENUM ('F', 'D', 'C', 'A', 'X', 'Y', 'Z')]
```

Resolving patch:
diff --git a/lib/rucio/db/sqla/session.py b/lib/rucio/db/sqla/session.py
--- a/lib/rucio/db/sqla/session.py
+++ b/lib/rucio/db/sqla/session.py
@@ -167,7 +167,7 @@ def my_on_connect(dbapi_con, connection_record):
     dbapi_con.action = caller
 
 
-def get_engine(echo=True):
+def get_engine():
     """ Creates a engine to a specific database.
         :returns: engine
     """
diff --git a/lib/rucio/db/sqla/util.py b/lib/rucio/db/sqla/util.py
--- a/lib/rucio/db/sqla/util.py
+++ b/lib/rucio/db/sqla/util.py
@@ -30,15 +30,16 @@
 from datetime import datetime
 from hashlib import sha256
 from os import urandom
-from traceback import format_exc
 from typing import TYPE_CHECKING
 
+import sqlalchemy
 from alembic import command, op
 from alembic.config import Config
-from sqlalchemy import func
-from sqlalchemy.engine import reflection
+from sqlalchemy import func, inspect
+from sqlalchemy.dialects.postgresql.base import PGInspector
 from sqlalchemy.exc import IntegrityError, DatabaseError
 from sqlalchemy.schema import CreateSchema, MetaData, Table, DropTable, ForeignKeyConstraint, DropConstraint
+from sqlalchemy.sql.ddl import DropSchema
 from sqlalchemy.sql.expression import select, text
 
 from rucio import alembicrevision
@@ -50,13 +51,14 @@
 from rucio.db.sqla.session import get_engine, get_session, get_dump_engine
 
 if TYPE_CHECKING:
-    from typing import Optional  # noqa: F401
+    from typing import Optional, Union  # noqa: F401
     from sqlalchemy.orm import Session  # noqa: F401
+    from sqlalchemy.engine import Inspector  # noqa: F401
 
 
-def build_database(echo=True):
+def build_database():
     """ Applies the schema to the database. Run this command once to build the database. """
-    engine = get_engine(echo=echo)
+    engine = get_engine()
 
     schema = config_get('database', 'schema', raise_exception=False)
     if schema:
@@ -79,9 +81,9 @@ def dump_schema():
     models.register_models(engine)
 
 
-def destroy_database(echo=True):
+def destroy_database():
     """ Removes the schema from the database. Only useful for test cases or malicious intents. """
-    engine = get_engine(echo=echo)
+    engine = get_engine()
 
     try:
         models.unregister_models(engine)
@@ -89,53 +91,43 @@ def destroy_database(echo=True):
         print('Cannot destroy schema -- assuming already gone, continuing:', e)
 
 
-def drop_everything(echo=True):
-    """ Pre-gather all named constraints and table names, and drop everything. This is better than using metadata.reflect();
-        metadata.drop_all() as it handles cyclical constraints between tables.
-        Ref. http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
+def drop_everything():
     """
-    engine = get_engine(echo=echo)
-    conn = engine.connect()
+    Pre-gather all named constraints and table names, and drop everything.
+    This is better than using metadata.reflect(); metadata.drop_all()
+    as it handles cyclical constraints between tables.
+    Ref. https://github.com/sqlalchemy/sqlalchemy/wiki/DropEverything
+    """
+    engine = get_engine()
 
     # the transaction only applies if the DB supports
     # transactional DDL, i.e. Postgresql, MS SQL Server
-    trans = conn.begin()
-
-    inspector = reflection.Inspector.from_engine(engine)
-
-    # gather all data first before dropping anything.
-    # some DBs lock after things have been dropped in
-    # a transaction.
-    metadata = MetaData()
-
-    tbs = []
-    all_fks = []
-
-    for table_name in inspector.get_table_names():
-        fks = []
-        for fk in inspector.get_foreign_keys(table_name):
-            if not fk['name']:
-                continue
-            fks.append(ForeignKeyConstraint((), (), name=fk['name']))
-        t = Table(table_name, metadata, *fks)
-        tbs.append(t)
-        all_fks.extend(fks)
-
-    for fkc in all_fks:
-        try:
-            print(str(DropConstraint(fkc)) + ';')
-            conn.execute(DropConstraint(fkc))
-        except:
-            print(format_exc())
-
-    for table in tbs:
-        try:
-            print(str(DropTable(table)).strip() + ';')
-            conn.execute(DropTable(table))
-        except:
-            print(format_exc())
-
-    trans.commit()
+    with engine.begin() as conn:
+
+        inspector = inspect(conn)  # type: Union[Inspector, PGInspector]
+
+        for tname, fkcs in reversed(
+                inspector.get_sorted_table_and_fkc_names(schema='*')):
+            if tname:
+                drop_table_stmt = DropTable(Table(tname, MetaData(), schema='*'))
+                conn.execute(drop_table_stmt)
+            elif fkcs:
+                if not engine.dialect.supports_alter:
+                    continue
+                for tname, fkc in fkcs:
+                    fk_constraint = ForeignKeyConstraint((), (), name=fkc)
+                    Table(tname, MetaData(), fk_constraint)
+                    drop_constraint_stmt = DropConstraint(fk_constraint)
+                    conn.execute(drop_constraint_stmt)
+
+        schema = config_get('database', 'schema', raise_exception=False)
+        if schema:
+            conn.execute(DropSchema(schema, cascade=True))
+
+        if engine.dialect.name == 'postgresql':
+            assert isinstance(inspector, PGInspector), 'expected a PGInspector'
+            for enum in inspector.get_enums(schema='*'):
+                sqlalchemy.Enum(**enum).drop(bind=conn)
 
 
 def create_base_vo():
diff --git a/tools/convert_database_vo.py b/tools/convert_database_vo.py
--- a/tools/convert_database_vo.py
+++ b/tools/convert_database_vo.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-# Copyright 2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2020-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,8 @@
 #
 # Authors:
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
-# - Gabriele Gaetano Fronze' <gabriele.fronze@to.infn.it>, 2020
+# - Gabriele Fronze' <gfronze@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
 
 import sys
 import os.path
@@ -64,7 +66,7 @@ def split_vo(dialect, column, return_vo=False):
             return func.SUBSTR(column, bindparam('int_1'), i - 1)
 
 
-def rename_vo(old_vo, new_vo, insert_new_vo=False, description=None, email=None, commit_changes=False, skip_history=False, echo=True):
+def rename_vo(old_vo, new_vo, insert_new_vo=False, description=None, email=None, commit_changes=False, skip_history=False):
     """
     Updates rows so that entries associated with `old_vo` are now associated with `new_vo` as part of multi-VO migration.
 
@@ -78,7 +80,7 @@ def rename_vo(old_vo, new_vo, insert_new_vo=False, description=None, email=None,
     :param skip_history:   If True then tables without FKC containing historical data will not be converted to save time.
     """
     success = True
-    engine = session.get_engine(echo=echo)
+    engine = session.get_engine()
     conn = engine.connect()
     trans = conn.begin()
     inspector = reflection.Inspector.from_engine(engine)
@@ -175,7 +177,7 @@ def rename_vo(old_vo, new_vo, insert_new_vo=False, description=None, email=None,
     return success
 
 
-def remove_vo(vo, commit_changes=False, skip_history=False, echo=True):
+def remove_vo(vo, commit_changes=False, skip_history=False):
     """
     Deletes rows associated with `vo` as part of multi-VO migration.
 
@@ -185,7 +187,7 @@ def remove_vo(vo, commit_changes=False, skip_history=False, echo=True):
     :param skip_history:   If True then tables without FKC containing historical data will not be converted to save time.
     """
     success = True
-    engine = session.get_engine(echo=echo)
+    engine = session.get_engine()
     conn = engine.connect()
     trans = conn.begin()
     inspector = reflection.Inspector.from_engine(engine)
@@ -299,7 +301,7 @@ def convert_to_mvo(new_vo, description, email, create_super_root=False, commit_c
         insert_new_vo = False
 
     success = rename_vo('def', new_vo, insert_new_vo=insert_new_vo, description=description, email=email,
-                        commit_changes=commit_changes, skip_history=skip_history, echo=echo)
+                        commit_changes=commit_changes, skip_history=skip_history)
     if create_super_root and success:
         create_root_account(create_counters=False)
     s.close()
@@ -320,13 +322,13 @@ def convert_to_svo(old_vo, delete_vos=False, commit_changes=False, skip_history=
         print('Multi-VO mode is not enabled in the config file, aborting conversion.')
         return
 
-    rename_vo(old_vo, 'def', commit_changes=commit_changes, skip_history=skip_history, echo=echo)
+    rename_vo(old_vo, 'def', commit_changes=commit_changes, skip_history=skip_history)
     s = session.get_session()
     if delete_vos:
         success_all = True
         for vo in list_vos(session=s):
             if vo['vo'] != 'def':
-                success = remove_vo(vo['vo'], commit_changes=commit_changes, skip_history=skip_history, echo=echo)
+                success = remove_vo(vo['vo'], commit_changes=commit_changes, skip_history=skip_history)
                 success_all = success_all and success
         if commit_changes and success_all:
             del_account(InternalAccount('super_root', vo='def'), session=s)


Test patch: