Repository: mathesar-foundation/mathesar

Issue:
Update the dependents query to get the type of the parent object and get rid of redundant columns
## Problem
Currently, the main dependents query returns some redundant columns (like identity) and doesn't return a type of the referenced object.

## Proposed solution
- Remove redundant columns from the query
- Add a referenced object type column
- Update types to validate the new response format

## Additional context
<!-- Add any other context or screenshots about the feature request here.-->



Discussion:

Resolving patch:
diff --git a/db/dependents/dependents_utils.py b/db/dependents/dependents_utils.py
--- a/db/dependents/dependents_utils.py
+++ b/db/dependents/dependents_utils.py
@@ -1,176 +1,177 @@
-from sqlalchemy import MetaData, Table, any_, case, column, exists, func, literal, select, text, true, union
+from sqlalchemy import MetaData, Table, any_, column, exists, func, literal, select, text, true, union
 from sqlalchemy.dialects.postgresql import array
 
 # OIDs assigned during normal database operation are constrained to be 16384 or higher.
 USER_DEFINED_OBJECTS_MIN_OID = 16384
+# automatic and normal dependents
+PG_DEPENDENT_TYPES = ['a', 'n']
+PG_CLASS_CATALOGUE_NAME = '\'pg_class\''
+START_LEVEL = 1
+MAX_LEVEL = 10
 
 
 def get_dependents_graph(referenced_object_id, engine):
-    all_dependent_objects_statement = _get_all_dependent_objects_statement(engine)
-    all_cte = all_dependent_objects_statement.cte(recursive=True, name='all')
+    dependency_pairs = _get_typed_dependency_pairs_stmt(engine)
+    dependency_pairs_cte = dependency_pairs.cte(recursive=True, name='dependency_pairs_cte')
 
     # anchor member which includes all dependents of a requested object
-    topq = select(
-        all_cte,
-        literal(1).label('level'),
-        array([all_cte.c.refobjid]).label('chain')).where(all_cte.c.refobjid == referenced_object_id).where(all_cte.c.objid != referenced_object_id)
-    topq = topq.cte('cte')
+    anchor = select(
+        dependency_pairs_cte,
+        literal(START_LEVEL).label('level'),
+        array([dependency_pairs_cte.c.refobjid]).label('dependency_chain')) \
+        .where(dependency_pairs_cte.c.refobjid == referenced_object_id) \
+        .where(dependency_pairs_cte.c.objid != referenced_object_id)
+    anchor = anchor.cte('cte')
 
     # recursive member which includes dependents for each object of the previous level
-    bottomq = select(
-        all_cte,
-        (topq.c.level + 1).label('level'),
-        topq.c.chain + array([all_cte.c.objid])) \
-        .where(topq.c.level < 10) \
-        .where(all_cte.c.objid != any_(topq.c.chain)) \
-        .where(all_cte.c.objid != all_cte.c.refobjid)
-    bottomq = bottomq.join(topq, all_cte.c.refobjid == topq.c.objid)
-
-    recursive_q = topq.union(bottomq)
-    q = select(recursive_q)
+    recursive = select(
+        dependency_pairs_cte,
+        (anchor.c.level + 1).label('level'),
+        anchor.c.dependency_chain + array([anchor.c.objid])) \
+        .where(anchor.c.level < MAX_LEVEL) \
+        .where(dependency_pairs_cte.c.objid != any_(anchor.c.dependency_chain)) \
+        .where(dependency_pairs_cte.c.objid != dependency_pairs_cte.c.refobjid)
+    recursive = recursive.join(anchor, dependency_pairs_cte.c.refobjid == anchor.c.objid)
+
+    recursive_stmt = anchor.union(recursive)
+    stmt = select(recursive_stmt)
 
     with engine.connect() as conn:
-        result = conn.execute(q)
-
-    # initial way of structuring the final response
-    # TODO: extract in a separate function
-    final = []
-    for r in result:
-        d = {}
-        d['level'] = r.level
-        d['obj'] = {'objid': r.objid, 'type': r.type}
-        d['parent_obj'] = {'objid': r.refobjid}
-        final.append(d)
+        result = conn.execute(stmt)
 
-    return final
+    return _get_structured_result(result)
 
 
 # finding table dependents based on foreign key constraints from the referenced tables
-def _get_table_dependents(foreign_key_dependents, pg_constraint):
-    pg_identify_object = select(
-        column("name"),
-        column("schema"),
-        column("type"),
-        column("identity")) \
-        .select_from(func.pg_identify_object(
-            text('\'pg_class\'::regclass::oid'),
-            pg_constraint.c.conrelid,
-            0)) \
-        .lateral()
+def _get_table_dependents(foreign_key_dependents, pg_constraint_table):
+    # TODO: update refobjsubid with actual values when working on columns
+    pg_identify_object = _get_pg_identify_object_lateral_stmt(
+        text(f'{PG_CLASS_CATALOGUE_NAME}::regclass::oid'), pg_constraint_table.c.conrelid, 0)
+
+    pg_identify_refobject = _get_pg_identify_object_lateral_stmt(
+        foreign_key_dependents.c.refclassid, foreign_key_dependents.c.refobjid, 0)
 
     # conrelid in this case is the oid of the table which a constraint resides in
     return select(
-        pg_constraint.c.conrelid.label('objid'),
+        foreign_key_dependents.c.classid,
+        pg_constraint_table.c.conrelid.label('objid'),
+        foreign_key_dependents.c.objsubid,
+        foreign_key_dependents.c.refclassid,
         foreign_key_dependents.c.refobjid,
-        foreign_key_dependents.c.column_number,
-        pg_identify_object.c.name,
-        pg_identify_object.c.schema,
-        pg_identify_object.c.type,
-        pg_identify_object.c.identity,
-        foreign_key_dependents.c.dependency_type) \
+        foreign_key_dependents.c.refobjsubid,
+        foreign_key_dependents.c.deptype,
+        pg_identify_object.c.name.label('objname'),
+        pg_identify_object.c.type.label('objtype'),
+        pg_identify_refobject.c.name.label('refobjname'),
+        pg_identify_refobject.c.type.label('refobjtype')) \
         .select_from(foreign_key_dependents) \
-        .join(pg_constraint, pg_constraint.c.oid == foreign_key_dependents.c.objid) \
+        .join(pg_constraint_table, pg_constraint_table.c.oid == foreign_key_dependents.c.objid) \
         .join(pg_identify_object, true()) \
-        .where(pg_constraint.c.confrelid != 0) \
+        .join(pg_identify_refobject, true()) \
+        .where(pg_constraint_table.c.confrelid != 0) \
         .group_by(
-            pg_constraint.c.conrelid,
-            foreign_key_dependents.c.refobjid,
-            foreign_key_dependents.c.column_number,
-            pg_identify_object.c.type,
+            foreign_key_dependents,
+            pg_constraint_table.c.conrelid,
             pg_identify_object.c.name,
-            pg_identify_object.c.schema,
-            pg_identify_object.c.identity,
-            foreign_key_dependents.c.dependency_type)
+            pg_identify_object.c.type,
+            pg_identify_refobject.c.name,
+            pg_identify_refobject.c.type)
 
 
-def _get_foreign_key_constraint_dependents(pg_identify_object, base):
-    return base.where(pg_identify_object.c.type == 'table constraint')
+def _get_foreign_key_constraint_dependents(pg_identify_object, dependency_pair):
+    return dependency_pair.where(pg_identify_object.c.type == 'table constraint')
 
 
 # getting a full list of dependents and identifying them
-def _get_all_dependent_objects_base_statement(pg_depend, pg_identify_object):
+def _get_dependency_pairs_stmt(pg_depend, pg_identify_object, pg_identify_refobject):
     result = select(
-        pg_depend.c.objid,
-        pg_depend.c.refobjid,
-        func.array_agg(pg_depend.c.objsubid).label('column_number'),
-        pg_identify_object.c.name,
-        pg_identify_object.c.schema,
-        pg_identify_object.c.type,
-        pg_identify_object.c.identity,
-        _get_dependency_case(pg_depend).label('dependency_type')) \
+        pg_depend,
+        pg_identify_object.c.name.label('objname'),
+        pg_identify_object.c.type.label('objtype'),
+        pg_identify_refobject.c.name.label('refobjname'),
+        pg_identify_refobject.c.type.label('refobjtype')) \
         .select_from(pg_depend) \
         .join(pg_identify_object, true()) \
-        .where(pg_depend.c.deptype == any_('{a,n}')) \
-        .where(pg_depend.c.objid >= 16384) \
+        .join(pg_identify_refobject, true()) \
+        .where(pg_depend.c.deptype == any_(array(PG_DEPENDENT_TYPES))) \
+        .where(pg_depend.c.objid >= USER_DEFINED_OBJECTS_MIN_OID) \
         .group_by(
-            pg_depend.c.objid,
-            pg_depend.c.refobjid,
-            pg_identify_object.c.type,
+            pg_depend,
             pg_identify_object.c.name,
-            pg_identify_object.c.schema,
-            pg_identify_object.c.identity,
-            pg_depend.c.deptype)
+            pg_identify_object.c.type,
+            pg_identify_refobject.c.name,
+            pg_identify_refobject.c.type)
 
     return result
 
 
-def _get_pg_depend(engine, metadata):
+def _get_pg_depend_table(engine, metadata):
     return Table("pg_depend", metadata, autoload_with=engine)
 
 
-def _get_pg_constraint(engine, metadata):
+def _get_pg_constraint_table(engine, metadata):
     return Table("pg_constraint", metadata, autoload_with=engine)
 
 
-def _get_pg_identify_object_lateral(source):
+def _get_pg_identify_object_lateral_stmt(classid, objid, objsubid):
     return select(
         column("name"),
-        column("schema"),
-        column("type"),
-        column("identity")) \
+        column("type")) \
         .select_from(func.pg_identify_object(
-            source.c.classid,
-            source.c.objid,
-            source.c.objsubid)) \
+            classid,
+            objid,
+            objsubid)) \
         .lateral()
 
 
-def _get_dependency_case(pg_depend):
-    return case(
-        (pg_depend.c.deptype == 'n', 'normal'),
-        (pg_depend.c.deptype == 'a', 'automatic')
-    ).label('dependency_type')
-
-
-def _get_all_dependent_objects_statement(engine):
+def _get_typed_dependency_pairs_stmt(engine):
     metadata = MetaData()
-    pg_depend = _get_pg_depend(engine, metadata)
-    pg_identify_object = _get_pg_identify_object_lateral(pg_depend)
-    pg_constraint = _get_pg_constraint(engine, metadata)
+
+    pg_depend = _get_pg_depend_table(engine, metadata)
+    pg_identify_object = _get_pg_identify_object_lateral_stmt(
+        pg_depend.c.classid, pg_depend.c.objid, pg_depend.c.objsubid)
+    pg_identify_refobject = _get_pg_identify_object_lateral_stmt(
+        pg_depend.c.refclassid, pg_depend.c.refobjid, 0)
+    pg_constraint = _get_pg_constraint_table(engine, metadata)
 
     # each statement filters the base statement extracting dependents of a specific type
     # so it's easy to exclude particular types or add new
-    base_stmt = _get_all_dependent_objects_base_statement(pg_depend, pg_identify_object)
-    foreign_key_constraint_dependents = _get_foreign_key_constraint_dependents(pg_identify_object, base_stmt).cte('foreign_key_constraint_dependents')
+    dependency_pair = _get_dependency_pairs_stmt(pg_depend, pg_identify_object, pg_identify_refobject)
+    foreign_key_constraint_dependents = _get_foreign_key_constraint_dependents(pg_identify_object, dependency_pair).cte('foreign_key_constraint_dependents')
     table_dependents = _get_table_dependents(foreign_key_constraint_dependents, pg_constraint).cte('table_dependents')
 
-    return union(select(foreign_key_constraint_dependents), select(table_dependents))
+    return union(
+        select(foreign_key_constraint_dependents),
+        select(table_dependents))
 
 
 def has_dependencies(referenced_object_id, engine):
     metadata = MetaData()
-    pg_depend = _get_pg_depend(engine, metadata)
+
+    pg_depend = _get_pg_depend_table(engine, metadata)
 
     stmt = select(
         exists(
             select().select_from(pg_depend)
             .where(pg_depend.c.refobjid == referenced_object_id)
-            .where(pg_depend.c.deptype == any_('{a,n}'))
+            .where(pg_depend.c.deptype == any_(array(PG_DEPENDENT_TYPES)))
             .where(pg_depend.c.objid >= USER_DEFINED_OBJECTS_MIN_OID)
         )
     )
 
     with engine.connect() as conn:
-        result = conn.execute(stmt)
+        result = conn.execute(stmt).scalar()
 
-    return result is not None
+    return result
+
+
+def _get_structured_result(dependency_graph_result):
+    result = []
+    for dependency_pair in dependency_graph_result:
+        d = {}
+        d['level'] = dependency_pair.level
+        d['obj'] = {'objid': dependency_pair.objid, 'type': dependency_pair.objtype}
+        d['parent_obj'] = {'objid': dependency_pair.refobjid, 'type': dependency_pair.refobjtype}
+        result.append(d)
+
+    return result
diff --git a/mathesar/api/serializers/dependents.py b/mathesar/api/serializers/dependents.py
--- a/mathesar/api/serializers/dependents.py
+++ b/mathesar/api/serializers/dependents.py
@@ -51,3 +51,4 @@ def get_mapping_field(self, data):
 class DependentSerializer(serializers.Serializer):
     obj = BaseDependentObjectSerializer()
     parent_obj = BaseDependentObjectSerializer()
+    level = serializers.IntegerField()


Test patch:
diff --git a/mathesar/tests/api/test_dependents_api.py b/mathesar/tests/api/test_dependents_api.py
--- a/mathesar/tests/api/test_dependents_api.py
+++ b/mathesar/tests/api/test_dependents_api.py
@@ -59,9 +59,9 @@ def test_dependents_response(two_foreign_key_tables, client):
     referer_foreign_key_constraint_id = [r['id'] for r in referrer_table_constraints_results if r['type'] == 'foreignkey']
 
     referent_expected_dependent_ids = referent_constraint_ids + referer_foreign_key_constraint_id + [referrer_table.id]
-    referent_actual_dependents_ids = [int(d['obj']['id']) for d in response_data if d['parent_obj']['objid'] == referent_table.oid]
+    referent_actual_dependents_ids = [int(d['obj']['id']) for d in response_data if int(d['parent_obj']['id']) == referent_table.id]
 
-    referrer_dependents_ids = [int(d['obj']['id']) for d in response_data if d['parent_obj']['objid'] == referrer_table.oid]
+    referrer_dependents_ids = [int(d['obj']['id']) for d in response_data if int(d['parent_obj']['id']) == referrer_table.id]
 
     assert sorted(referent_expected_dependent_ids) == sorted(referent_actual_dependents_ids)
     assert sorted(referer_constraint_ids) == sorted(referrer_dependents_ids)