Repository: apache/airflow

Issue:
need to run airflow cli command "airflow dags details" for multiple DAG IDs
### Description

Hello Team,

Hope you are doing well!

We have one use case where we want to check last parsed time for the multiple DAGs. Currently we have one airflow cli command i.e "airflow dags details" which gives the output for only one DAG ID but we want same result of this cli command for multiple DAG IDs.

We would you like to request you to create similar or modify the same cli command for giving last parsed time for multiple tasks.


Thanks & Regards,
Vidya Panchal 

### Use case/motivation

cli command : airflow dags details ----> give as output for only one DAG ID/DAG name.

We want "airflow dags details" cli command also give result for multiple DAG IDs / DAG names. 

### Related issues

We have one business use case where we want to fetch last parsed time for multiple DAGs and perform our business operations accordingly. 

### Are you willing to submit a PR?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)



Discussion:
I'm happy to pick this up if we want to move forward with this suggestion. 
Maybe better to add this as output for `airflow dags list` ?
I agree adding to `airflow dags list` as additional optional column makes a good sense. Feel free to contribute, probably a good small contribution, entry point could be `airflow/cli/commands/dag_command.py:dag_list_dags()` and adding a new CLI parameter
Thank you for acknowledging it. Can we also have similar implementation for REST API where we get DAGs parsing details for multiple DAGs in single API call?
Team,

Any update on this task ? We need to get the last parsed time information of multiple DAGs at the same time. Right now this would mean running the CLI command multiple times with different DAG IDs. This enhancement will prevent the multiple command executions. Also, as Vidya mentioned, we need similar support in the REST API as well.

Regards,
Sidrah.
Hi @sidrahsayyad you are polling for status in an Open Source project. You know that people contribute to this sometimes as full time job but mostly in spare time or as side business? There is no SLA behind a feature request.

If you are under pressure then please arrange a support agreement with a commercial vendor to support you on this or feel free to contribute.

Airflow does not provide an SLA for having requested features being implemented.

Resolving patch:
diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py
--- a/airflow/cli/cli_config.py
+++ b/airflow/cli/cli_config.py
@@ -986,6 +986,13 @@ def string_lower_type(val):
     help="If passed, serialized DAGs will be cleared but not reserialized.",
 )
 
+ARG_DAG_LIST_COLUMNS = Arg(
+    ("--columns",),
+    type=string_list_type,
+    help="List of columns to render. (default: ['dag_id', 'fileloc', 'owner', 'is_paused'])",
+    default=("dag_id", "fileloc", "owners", "is_paused"),
+)
+
 ALTERNATIVE_CONN_SPECS_ARGS = [
     ARG_CONN_TYPE,
     ARG_CONN_DESCRIPTION,
@@ -1032,7 +1039,7 @@ class GroupCommand(NamedTuple):
         name="list",
         help="List all the DAGs",
         func=lazy_load_command("airflow.cli.commands.dag_command.dag_list_dags"),
-        args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
+        args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE, ARG_DAG_LIST_COLUMNS),
     ),
     ActionCommand(
         name="list-import-errors",
diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py
--- a/airflow/cli/commands/dag_command.py
+++ b/airflow/cli/commands/dag_command.py
@@ -351,8 +351,20 @@ def print_execution_interval(interval: DataInterval | None):
 @cli_utils.action_cli
 @suppress_logs_and_warning
 @providers_configuration_loaded
-def dag_list_dags(args) -> None:
+@provide_session
+def dag_list_dags(args, session=NEW_SESSION) -> None:
     """Display dags with or without stats at the command line."""
+    cols = args.columns if args.columns else []
+    invalid_cols = [c for c in cols if c not in dag_schema.fields]
+    valid_cols = [c for c in cols if c in dag_schema.fields]
+    if invalid_cols:
+        from rich import print as rich_print
+
+        rich_print(
+            f"[red][bold]Error:[/bold] Ignoring the following invalid columns: {invalid_cols}.  "
+            f"List of valid columns: {list(dag_schema.fields.keys())}",
+            file=sys.stderr,
+        )
     dagbag = DagBag(process_subdir(args.subdir))
     if dagbag.import_errors:
         from rich import print as rich_print
@@ -362,15 +374,16 @@ def dag_list_dags(args) -> None:
             "For details, run `airflow dags list-import-errors`",
             file=sys.stderr,
         )
+
+    def get_dag_detail(dag: DAG) -> dict:
+        dag_model = DagModel.get_dagmodel(dag.dag_id, session=session)
+        dag_detail = dag_schema.dump(dag_model)
+        return {col: dag_detail[col] for col in valid_cols}
+
     AirflowConsole().print_as(
         data=sorted(dagbag.dags.values(), key=operator.attrgetter("dag_id")),
         output=args.output,
-        mapper=lambda x: {
-            "dag_id": x.dag_id,
-            "filepath": x.filepath,
-            "owner": x.owner,
-            "paused": x.get_is_paused(),
-        },
+        mapper=get_dag_detail,
     )
 
 


Test patch:
diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py
--- a/tests/cli/commands/test_dag_command.py
+++ b/tests/cli/commands/test_dag_command.py
@@ -524,15 +524,36 @@ def test_cli_get_dag_details(self):
 
     @conf_vars({("core", "load_examples"): "true"})
     def test_cli_list_dags(self):
-        args = self.parser.parse_args(["dags", "list", "--output", "yaml"])
+        args = self.parser.parse_args(["dags", "list", "--output", "json"])
         with contextlib.redirect_stdout(StringIO()) as temp_stdout:
             dag_command.dag_list_dags(args)
             out = temp_stdout.getvalue()
-        assert "owner" in out
-        assert "airflow" in out
-        assert "paused" in out
-        assert "airflow/example_dags/example_complex.py" in out
-        assert "- dag_id:" in out
+            dag_list = json.loads(out)
+        for key in ["dag_id", "fileloc", "owners", "is_paused"]:
+            assert key in dag_list[0]
+        assert any("airflow/example_dags/example_complex.py" in d["fileloc"] for d in dag_list)
+
+    @conf_vars({("core", "load_examples"): "true"})
+    def test_cli_list_dags_custom_cols(self):
+        args = self.parser.parse_args(
+            ["dags", "list", "--output", "json", "--columns", "dag_id,last_parsed_time"]
+        )
+        with contextlib.redirect_stdout(StringIO()) as temp_stdout:
+            dag_command.dag_list_dags(args)
+            out = temp_stdout.getvalue()
+            dag_list = json.loads(out)
+        for key in ["dag_id", "last_parsed_time"]:
+            assert key in dag_list[0]
+        for key in ["fileloc", "owners", "is_paused"]:
+            assert key not in dag_list[0]
+
+    @conf_vars({("core", "load_examples"): "true"})
+    def test_cli_list_dags_invalid_cols(self):
+        args = self.parser.parse_args(["dags", "list", "--output", "json", "--columns", "dag_id,invalid_col"])
+        with contextlib.redirect_stderr(StringIO()) as temp_stderr:
+            dag_command.dag_list_dags(args)
+            out = temp_stderr.getvalue()
+        assert "Ignoring the following invalid columns: ['invalid_col']" in out
 
     @conf_vars({("core", "load_examples"): "false"})
     def test_cli_list_dags_prints_import_errors(self):