Repository: apache/airflow

Issue:
exception when root account goes to http://airflow.ordercapital.com/dag-dependencies
Happens every time 


Python version: 3.8.10
Airflow version: 2.1.0
Node: airflow-web-55974db849-5bdxq
-------------------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/flask/app.py", line 2447, in wsgi_app
    response = self.full_dispatch_request()
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/flask/app.py", line 1952, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/flask/app.py", line 1821, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/flask/_compat.py", line 39, in reraise
    raise value
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/flask/app.py", line 1950, in full_dispatch_request
    rv = self.dispatch_request()
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/flask/app.py", line 1936, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/www/auth.py", line 34, in decorated
    return func(*args, **kwargs)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/www/decorators.py", line 97, in view_func
    return f(*args, **kwargs)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/www/decorators.py", line 60, in wrapper
    return f(*args, **kwargs)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/www/views.py", line 4004, in list
    self._calculate_graph()
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/www/views.py", line 4023, in _calculate_graph
    for dag, dependencies in SerializedDagModel.get_dag_dependencies().items():
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/utils/session.py", line 70, in wrapper
    return func(*args, session=session, **kwargs)
  File "/opt/bitnami/airflow/venv/lib/python3.8/site-packages/airflow/models/serialized_dag.py", line 321, in get_dag_dependencies
    dependencies[row[0]] = [DagDependency(**d) for d in row[1]]
TypeError: 'NoneType' object is not iterable






Discussion:
Thanks for opening your first issue here! Be sure to follow the issue template!

I'm guessing this might be related to #16328. Different traceback, but might be the same root cause?
Actually no. The error message suggests `row[1]` is `None`, which is a value returned by Postgres’s `json_extract_path`:

https://github.com/apache/airflow/blob/304e174674ff6921cb7ed79c0158949b50eff8fe/airflow/models/serialized_dag.py#L318-L321

I *think* this means `data["dag"]["dag_dependencies"]` is not found or None? This is a new field in 2.1.0, but I think we didn’t migrate previously-serialised DAG rows correctly. Reading its original implementation (#13199), I think we should either add a data migration to backfill previously-serialised rows, or just pretend those DAGs don’t have dependencies.
@uranusjr Just assume no dependencies -- the serialized dag _should_ get updated quickly enough by the scheduler anyway.

Ohh, I wonder if this happens if _no_ dags have any dependencies.
@apyshkin What database are you using?

Resolving patch:
diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py
--- a/airflow/models/serialized_dag.py
+++ b/airflow/models/serialized_dag.py
@@ -313,18 +313,12 @@ def get_dag_dependencies(cls, session: Session = None) -> Dict[str, List['DagDep
         :param session: ORM Session
         :type session: Session
         """
-        dependencies = {}
-
         if session.bind.dialect.name in ["sqlite", "mysql"]:
-            for row in session.query(cls.dag_id, func.json_extract(cls.data, "$.dag.dag_dependencies")).all():
-                dependencies[row[0]] = [DagDependency(**d) for d in json.loads(row[1])]
+            query = session.query(cls.dag_id, func.json_extract(cls.data, "$.dag.dag_dependencies"))
+            iterator = ((dag_id, json.loads(deps_data) if deps_data else []) for dag_id, deps_data in query)
         elif session.bind.dialect.name == "mssql":
-            for row in session.query(cls.dag_id, func.json_query(cls.data, "$.dag.dag_dependencies")).all():
-                dependencies[row[0]] = [DagDependency(**d) for d in json.loads(row[1])]
+            query = session.query(cls.dag_id, func.json_query(cls.data, "$.dag.dag_dependencies"))
+            iterator = ((dag_id, json.loads(deps_data) if deps_data else []) for dag_id, deps_data in query)
         else:
-            for row in session.query(
-                cls.dag_id, func.json_extract_path(cls.data, "dag", "dag_dependencies")
-            ).all():
-                dependencies[row[0]] = [DagDependency(**d) for d in row[1]]
-
-        return dependencies
+            iterator = session.query(cls.dag_id, func.json_extract_path(cls.data, "dag", "dag_dependencies"))
+        return {dag_id: [DagDependency(**d) for d in (deps_data or [])] for dag_id, deps_data in iterator}


Test patch:
diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py
--- a/tests/models/test_serialized_dag.py
+++ b/tests/models/test_serialized_dag.py
@@ -20,6 +20,8 @@
 
 import unittest
 
+from parameterized import parameterized
+
 from airflow import DAG, example_dags as example_dags_module
 from airflow.models import DagBag
 from airflow.models.dagcode import DagCode
@@ -149,3 +151,19 @@ def test_bulk_sync_to_db(self):
         ]
         with assert_queries_count(10):
             SDM.bulk_sync_to_db(dags)
+
+    @parameterized.expand([({"dag_dependencies": None},), ({},)])
+    def test_get_dag_dependencies_default_to_empty(self, dag_dependencies_fields):
+        """Test a pre-2.1.0 serialized DAG can deserialize DAG dependencies."""
+        example_dags = make_example_dags(example_dags_module)
+
+        with create_session() as session:
+            sdms = [SDM(dag) for dag in example_dags.values()]
+            # Simulate pre-2.1.0 format.
+            for sdm in sdms:
+                del sdm.data["dag"]["dag_dependencies"]
+                sdm.data["dag"].update(dag_dependencies_fields)
+            session.bulk_save_objects(sdms)
+
+        expected_dependencies = {dag_id: [] for dag_id in example_dags}
+        assert SDM.get_dag_dependencies() == expected_dependencies