Repository: astronomer/dag-factory

Issue:
Review broad Exception and remove this pattern, where possible
DAG Factory captures broad exceptions in many places, which makes it quite hard to troubleshoot issues:
```
(venv) ➜  dag-factory git:(fmain) # grep -rn "except Exception" dagfactory
dagfactory/dagbuilder.py:150:        except Exception as err:
dagfactory/dagbuilder.py:296:        except Exception as err:
dagfactory/dagbuilder.py:300:        except Exception as err:
dagfactory/dagbuilder.py:317:        except Exception as err:
dagfactory/dagbuilder.py:561:        except Exception as err:
dagfactory/utils.py:33:    except Exception as err:
dagfactory/dagfactory.py:74:        except Exception as err:
dagfactory/dagfactory.py:111:            except Exception as err:
dagfactory/dagfactory.py:207:        except Exception:  # pylint: disable=broad-except
```

The goal of this PR is to address this, removing this pattern where possible.
Check possibility to capture a more specific exception
Address: https://github.com/astronomer/dag-factory/pull/407#discussion_r2109562413

At many places, we are capturing `Exception` investigate if we can capture specific exception

<img width="1644" alt="Image" src="https://github.com/user-attachments/assets/cb644096-dacb-46ad-9c74-50f3cc0d9516" />


Discussion:

Patch:
diff --git a/dagfactory/dagbuilder.py b/dagfactory/dagbuilder.py
index b14e0c45..aee80cb7 100644
--- a/dagfactory/dagbuilder.py
+++ b/dagfactory/dagbuilder.py
@@ -106,10 +106,7 @@ def get_dag_params(self) -> Dict[str, Any]:
 
         :returns: dict of dag parameters
         """
-        try:
-            dag_params: Dict[str, Any] = utils.merge_configs(self.dag_config, self.default_config)
-        except Exception as err:
-            raise DagFactoryConfigException("Failed to merge config with default config") from err
+        dag_params: Dict[str, Any] = utils.merge_configs(self.dag_config, self.default_config)
         dag_params["dag_id"]: str = self.dag_name
 
         # If there are no default_args, add an empty dictionary
@@ -216,15 +213,9 @@ def make_timetable(timetable: str, timetable_params: Dict[str, Any]) -> Timetabl
 
         :returns instance of timetable object
         """
-        try:
-            # class is a Callable https://stackoverflow.com/a/34578836/3679900
-            timetable_obj: Callable[..., Timetable] = import_string(timetable)
-        except Exception as err:
-            raise DagFactoryException(f"Failed to import timetable {timetable} due to: {err}") from err
-        try:
-            schedule: Timetable = timetable_obj(**timetable_params)
-        except Exception as err:  # pragma: no cover
-            raise DagFactoryException(f"Failed to create {timetable_obj} due to: {err}") from err
+        # class is a Callable https://stackoverflow.com/a/34578836/3679900
+        timetable_obj: Callable[..., Timetable] = import_string(timetable)
+        schedule: Timetable = timetable_obj(**timetable_params)
         return schedule
 
     @staticmethod
@@ -273,93 +264,83 @@ def make_task(operator: str, task_params: Dict[str, Any]) -> BaseOperator:
 
         :returns: instance of operator object
         """
-        try:
-            # class is a Callable https://stackoverflow.com/a/34578836/3679900
-            operator_obj: Callable[..., BaseOperator] = import_string(operator)
-        except Exception as err:
-            raise DagFactoryException(f"Failed to import operator: {operator}") from err
+        # class is a Callable https://stackoverflow.com/a/34578836/3679900
+        operator_obj: Callable[..., BaseOperator] = import_string(operator)
         # pylint: disable=too-many-nested-blocks
-        try:
-            if issubclass(operator_obj, (PythonOperator, BranchPythonOperator, PythonSensor)):
-                if (
-                    not task_params.get("python_callable")
-                    and not task_params.get("python_callable_name")
-                    and not task_params.get("python_callable_file")
-                ):
-                    # pylint: disable=line-too-long
-                    raise DagFactoryException(
-                        "Failed to create task. PythonOperator, BranchPythonOperator and PythonSensor requires \
-                        `python_callable_name` and `python_callable_file` "
-                        "parameters.\nOptionally you can load python_callable "
-                        "from a file. with the special pyyaml notation:\n"
-                        "  python_callable_file: !!python/name:my_module.my_func"
-                    )
-                if not task_params.get("python_callable"):
-                    task_params["python_callable"]: Callable = utils.get_python_callable(
-                        task_params["python_callable_name"], task_params["python_callable_file"]
-                    )
-                    # remove dag-factory specific parameters
-                    # Airflow 2.0 doesn't allow these to be passed to operator
-                    del task_params["python_callable_name"]
-                    del task_params["python_callable_file"]
-                elif isinstance(task_params["python_callable"], str):
-                    task_params["python_callable"]: Callable = import_string(task_params["python_callable"])
-
-            # Check for the custom success and failure callables in SqlSensor. These are considered
-            # optional, so no failures in case they aren't found. Note: there's no reason to
-            # declare both a callable file and a lambda function for success/failure parameter.
-            # If both are found the object will not throw and error, instead callable file will
-            # take precedence over the lambda function
-            if SQL_SENSOR_CLASS and issubclass(operator_obj, SQL_SENSOR_CLASS):
-                # Success checks
-                if task_params.get("success_check_file") and task_params.get("success_check_name"):
-                    task_params["success"]: Callable = utils.get_python_callable(
-                        task_params["success_check_name"], task_params["success_check_file"]
-                    )
-                    del task_params["success_check_name"]
-                    del task_params["success_check_file"]
-                elif task_params.get("success_check_lambda"):
-                    task_params["success"]: Callable = utils.get_python_callable_lambda(
-                        task_params["success_check_lambda"]
-                    )
-                    del task_params["success_check_lambda"]
-                # Failure checks
-                if task_params.get("failure_check_file") and task_params.get("failure_check_name"):
-                    task_params["failure"]: Callable = utils.get_python_callable(
-                        task_params["failure_check_name"], task_params["failure_check_file"]
-                    )
-                    del task_params["failure_check_name"]
-                    del task_params["failure_check_file"]
-                elif task_params.get("failure_check_lambda"):
-                    task_params["failure"]: Callable = utils.get_python_callable_lambda(
-                        task_params["failure_check_lambda"]
-                    )
-                    del task_params["failure_check_lambda"]
-
-            # Only handle HTTP operator/sensor if the package is installed
-            if (HTTP_OPERATOR_CLASS or HTTP_SENSOR_CLASS) and issubclass(
-                operator_obj, (HTTP_OPERATOR_CLASS, HTTP_SENSOR_CLASS)
+        if issubclass(operator_obj, (PythonOperator, BranchPythonOperator, PythonSensor)):
+            if (
+                not task_params.get("python_callable")
+                and not task_params.get("python_callable_name")
+                and not task_params.get("python_callable_file")
             ):
-                task_params = DagBuilder._handle_http_sensor(operator_obj, task_params)
+                # pylint: disable=line-too-long
+                raise DagFactoryException(
+                    "Failed to create task. PythonOperator, BranchPythonOperator and PythonSensor requires \
+                    `python_callable_name` and `python_callable_file` "
+                    "parameters.\nOptionally you can load python_callable "
+                    "from a file. with the special pyyaml notation:\n"
+                    "  python_callable_file: !!python/name:my_module.my_func"
+                )
+            if not task_params.get("python_callable"):
+                task_params["python_callable"]: Callable = utils.get_python_callable(
+                    task_params["python_callable_name"], task_params["python_callable_file"]
+                )
+                # remove dag-factory specific parameters
+                # Airflow 2.0 doesn't allow these to be passed to operator
+                del task_params["python_callable_name"]
+                del task_params["python_callable_file"]
+            elif isinstance(task_params["python_callable"], str):
+                task_params["python_callable"]: Callable = import_string(task_params["python_callable"])
+
+        # Check for the custom success and failure callables in SqlSensor. These are considered
+        # optional, so no failures in case they aren't found. Note: there's no reason to
+        # declare both a callable file and a lambda function for success/failure parameter.
+        # If both are found the object will not throw and error, instead callable file will
+        # take precedence over the lambda function
+        if SQL_SENSOR_CLASS and issubclass(operator_obj, SQL_SENSOR_CLASS):
+            # Success checks
+            if task_params.get("success_check_file") and task_params.get("success_check_name"):
+                task_params["success"]: Callable = utils.get_python_callable(
+                    task_params["success_check_name"], task_params["success_check_file"]
+                )
+                del task_params["success_check_name"]
+                del task_params["success_check_file"]
+            elif task_params.get("success_check_lambda"):
+                task_params["success"]: Callable = utils.get_python_callable_lambda(task_params["success_check_lambda"])
+                del task_params["success_check_lambda"]
+            # Failure checks
+            if task_params.get("failure_check_file") and task_params.get("failure_check_name"):
+                task_params["failure"]: Callable = utils.get_python_callable(
+                    task_params["failure_check_name"], task_params["failure_check_file"]
+                )
+                del task_params["failure_check_name"]
+                del task_params["failure_check_file"]
+            elif task_params.get("failure_check_lambda"):
+                task_params["failure"]: Callable = utils.get_python_callable_lambda(task_params["failure_check_lambda"])
+                del task_params["failure_check_lambda"]
+
+        # Only handle HTTP operator/sensor if the package is installed
+        if (HTTP_OPERATOR_CLASS or HTTP_SENSOR_CLASS) and issubclass(
+            operator_obj, (HTTP_OPERATOR_CLASS, HTTP_SENSOR_CLASS)
+        ):
+            task_params = DagBuilder._handle_http_sensor(operator_obj, task_params)
 
-            DagBuilder.adjust_general_task_params(task_params)
+        DagBuilder.adjust_general_task_params(task_params)
 
-            expand_kwargs: Dict[str, Union[Dict[str, Any], Any]] = {}
-            if utils.check_dict_key(task_params, "expand") or utils.check_dict_key(task_params, "partial"):
-                # Getting expand and partial kwargs from task_params
-                (task_params, expand_kwargs, partial_kwargs) = utils.get_expand_partial_kwargs(task_params)
+        expand_kwargs: Dict[str, Union[Dict[str, Any], Any]] = {}
+        if utils.check_dict_key(task_params, "expand") or utils.check_dict_key(task_params, "partial"):
+            # Getting expand and partial kwargs from task_params
+            (task_params, expand_kwargs, partial_kwargs) = utils.get_expand_partial_kwargs(task_params)
 
-                # If there are partial_kwargs we should merge them with existing task_params
-                if partial_kwargs and not utils.is_partial_duplicated(partial_kwargs, task_params):
-                    task_params.update(partial_kwargs)
+            # If there are partial_kwargs we should merge them with existing task_params
+            if partial_kwargs and not utils.is_partial_duplicated(partial_kwargs, task_params):
+                task_params.update(partial_kwargs)
 
-            task: Union[BaseOperator, MappedOperator] = (
-                operator_obj(**task_params)
-                if not expand_kwargs
-                else operator_obj.partial(**task_params).expand(**expand_kwargs)
-            )
-        except Exception as err:
-            raise DagFactoryException(f"Failed to create {operator_obj} task: {err}") from err
+        task: Union[BaseOperator, MappedOperator] = (
+            operator_obj(**task_params)
+            if not expand_kwargs
+            else operator_obj.partial(**task_params).expand(**expand_kwargs)
+        )
         return task
 
     @staticmethod
diff --git a/dagfactory/dagfactory.py b/dagfactory/dagfactory.py
index 5aee6c57..92a2b2c1 100644
--- a/dagfactory/dagfactory.py
+++ b/dagfactory/dagfactory.py
@@ -199,16 +199,13 @@ def _load_dag_config(self, config_filepath: str) -> Dict[str, Any]:
         :returns: dict from YAML config file
         """
         # pylint: disable=consider-using-with
-        try:
-            config = load_yaml_file(config_filepath)
+        config = load_yaml_file(config_filepath)
 
-            # This will only invoke in the CI
-            # Make yaml DAG compatible for Airflow 3
-            if version.parse(AIRFLOW_VERSION) >= version.parse("3.0.0") and os.getenv("AUTO_CONVERT_TO_AF3"):
-                config = update_yaml_structure(config)
+        # This will only invoke in the CI
+        # Make yaml DAG compatible for Airflow 3
+        if version.parse(AIRFLOW_VERSION) >= version.parse("3.0.0") and os.getenv("AUTO_CONVERT_TO_AF3"):
+            config = update_yaml_structure(config)
 
-        except Exception as err:
-            raise DagFactoryConfigException(f"Invalid DAG Factory config file: {err}")
         return config
 
     def get_dag_configs(self) -> Dict[str, Dict[str, Any]]:
@@ -257,11 +254,8 @@ def build_dags(self) -> Dict[str, DAG]:
                 default_config=default_config,
                 yml_dag=self._serialise_config_md(dag_name, dag_config, default_config),
             )
-            try:
-                dag: Dict[str, Union[str, DAG]] = dag_builder.build()
-                dags[dag["dag_id"]]: DAG = dag["dag"]
-            except Exception as err:
-                raise DagFactoryException(f"Failed to generate dag {dag_name}: {err}") from err
+            dag: Dict[str, Union[str, DAG]] = dag_builder.build()
+            dags[dag["dag_id"]]: DAG = dag["dag"]
 
         return dags
 
diff --git a/dagfactory/utils.py b/dagfactory/utils.py
index df2053fc..572c445c 100644
--- a/dagfactory/utils.py
+++ b/dagfactory/utils.py
@@ -40,10 +40,7 @@ def get_datetime(date_value: Union[str, datetime, date], timezone: str = "UTC")
     :returns: datetime for date_value
     :type: datetime.datetime
     """
-    try:
-        local_tz: pendulum.timezone = pendulum.timezone(timezone)
-    except Exception as err:
-        raise DagFactoryException("Failed to create timezone") from err
+    local_tz: pendulum.timezone = pendulum.timezone(timezone)
     if isinstance(date_value, datetime):
         return date_value.replace(tzinfo=local_tz)
     if isinstance(date_value, date):
diff --git a/dev/dags/sample.py b/dev/dags/sample.py
index 2c9c1d84..ce6453bc 100644
--- a/dev/dags/sample.py
+++ b/dev/dags/sample.py
@@ -1,5 +1,5 @@
-import os
 import csv
+import os
 from datetime import datetime, timedelta
 from random import randint
 from typing import Any
@@ -9,11 +9,6 @@
 except ImportError:
     from airflow.operators.python import get_current_context
 
-try:
-    from airflow.sdk import ObjectStoragePath
-except ImportError:
-    from airflow.io.path import ObjectStoragePath
-
 
 def build_numbers_list():
     return [2, 4, 6]
@@ -77,8 +72,13 @@ def generate_data():
 
     print(f"Produced data to file://{file_path}")
 
-    
-def object_storage_ops(my_obj_storage: ObjectStoragePath) -> None:
+
+def object_storage_ops(my_obj_storage) -> None:
+    try:
+        from airflow.sdk import ObjectStoragePath
+    except ImportError:
+        from airflow.io.path import ObjectStoragePath
+
     assert isinstance(my_obj_storage, ObjectStoragePath)
     with my_obj_storage.open("rb") as f:
         text = f.read().decode("utf-8")


Test patch:
diff --git a/tests/test_dagbuilder_httpoperator.py b/tests/test_dagbuilder_httpoperator.py
index ae1527a8..c5ca3e91 100644
--- a/tests/test_dagbuilder_httpoperator.py
+++ b/tests/test_dagbuilder_httpoperator.py
@@ -13,7 +13,6 @@
     from airflow.models.dag import DAG  # noqa: F401
 
 from dagfactory.dagbuilder import DagBuilder
-from dagfactory.exceptions import DagFactoryException
 from tests.utils import get_schedule_key
 
 # Get current directory and project root
@@ -165,7 +164,7 @@ def test_http_operator_with_invalid_json_string(invalid_json):
         "data": invalid_json,
     }
 
-    with pytest.raises(DagFactoryException):
+    with pytest.raises(ValueError):
         td.make_task(HTTP_OPERATOR_PATH, task_params)