Repository: dbt-labs/dbt-core

Issue:
[CT-352] Catch all Requests Exception on Deps Install and Retry
We are not retrying for all exceptions from Requests and it's causing more unexpected/seemingly unexplained failures.  We should try to mitigate these failures by retrying (up to 5 times).  The logic to retry already exists.  We just need to make sure we catch the right/entirety of exceptions.

1. Catch *all* exceptions from Requests by catching just `requests.exceptions.RequestException` since all exceptions that Requests explicitly raises inherit from [`requests.exceptions.RequestException`](https://docs.python-requests.org/en/latest/api/#requests.RequestException).  Note that the `ReadError` should remain.

2. Also fire a new event to track what exception we're retrying so that we can track where the failures are in the future.

https://github.com/dbt-labs/dbt-core/blob/ecfd77f1caa10b7b15ca01104bcf3a9b54f3bb89/core/dbt/utils.py#L609-L620

Related Issues: #4601, #4178 


Discussion:
@emmyoop I'm just noting this down here in case it gets lost in our process. We saw another surge in `dbt deps` command failing because of hub returning 503 errors. 

If we are catching and retrying on `RequestException`, we should be able catch the hub's 503 errors along with other HTTP error codes. 

Resolving patch:
diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py
--- a/core/dbt/events/types.py
+++ b/core/dbt/events/types.py
@@ -2346,7 +2346,7 @@ def message(self) -> str:
 class RetryExternalCall(DebugLevel):
     attempt: int
     max: int
-    code: str = "Z045"
+    code: str = "M020"
 
     def message(self) -> str:
         return f"Retrying external call. Attempt: {self.attempt} Max attempts: {self.max}"
@@ -2384,6 +2384,15 @@ def message(self) -> str:
         return "Internal event buffer full. Earliest events will be dropped (FIFO)."
 
 
+@dataclass
+class RecordRetryException(DebugLevel):
+    exc: Exception
+    code: str = "M021"
+
+    def message(self) -> str:
+        return f"External call exception: {self.exc}"
+
+
 # since mypy doesn't run on every file we need to suggest to mypy that every
 # class gets instantiated. But we don't actually want to run this code.
 # making the conditional `if False` causes mypy to skip it as dead code so
@@ -2737,3 +2746,4 @@ def message(self) -> str:
     GeneralWarningMsg(msg="", log_fmt="")
     GeneralWarningException(exc=Exception(""), log_fmt="")
     EventBufferFull()
+    RecordRetryException(exc=Exception(""))
diff --git a/core/dbt/utils.py b/core/dbt/utils.py
--- a/core/dbt/utils.py
+++ b/core/dbt/utils.py
@@ -17,7 +17,7 @@
 from contextlib import contextmanager
 from dbt.exceptions import ConnectionException
 from dbt.events.functions import fire_event
-from dbt.events.types import RetryExternalCall
+from dbt.events.types import RetryExternalCall, RecordRetryException
 from dbt import flags
 from enum import Enum
 from typing_extensions import Protocol
@@ -600,19 +600,19 @@ def __contains__(self, name) -> bool:
 
 def _connection_exception_retry(fn, max_attempts: int, attempt: int = 0):
     """Attempts to run a function that makes an external call, if the call fails
-    on a connection error, timeout or decompression issue, it will be tried up to 5 more times.
-    See https://github.com/dbt-labs/dbt-core/issues/4579 for context on this decompression issues
-    specifically.
+    on a Requests exception or decompression issue (ReadError), it will be tried
+    up to 5 more times.  All exceptions that Requests explicitly raises inherit from
+    requests.exceptions.RequestException.  See https://github.com/dbt-labs/dbt-core/issues/4579
+    for context on this decompression issues specifically.
     """
     try:
         return fn()
     except (
-        requests.exceptions.ConnectionError,
-        requests.exceptions.Timeout,
-        requests.exceptions.ContentDecodingError,
+        requests.exceptions.RequestException,
         ReadError,
     ) as exc:
         if attempt <= max_attempts - 1:
+            fire_event(RecordRetryException(exc=exc))
             fire_event(RetryExternalCall(attempt=attempt, max=max_attempts))
             time.sleep(1)
             _connection_exception_retry(fn, max_attempts, attempt + 1)


Test patch:
diff --git a/test/unit/test_core_dbt_utils.py b/test/unit/test_core_dbt_utils.py
--- a/test/unit/test_core_dbt_utils.py
+++ b/test/unit/test_core_dbt_utils.py
@@ -12,27 +12,17 @@ def test_connection_exception_retry_none(self):
         connection_exception_retry(lambda: Counter._add(), 5)
         self.assertEqual(1, counter)
 
+    def test_connection_exception_retry_success_requests_exception(self):
+        Counter._reset()
+        connection_exception_retry(lambda: Counter._add_with_requests_exception(), 5)
+        self.assertEqual(2, counter) # 2 = original attempt returned None, plus 1 retry
+
     def test_connection_exception_retry_max(self):
         Counter._reset()
         with self.assertRaises(ConnectionException):
             connection_exception_retry(lambda: Counter._add_with_exception(), 5)
         self.assertEqual(6, counter) # 6 = original attempt plus 5 retries
 
-    def test_connection_exception_retry_success(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add_with_limited_exception(), 5)
-        self.assertEqual(2, counter) # 2 = original attempt plus 1 retry
-
-    def test_connection_timeout(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add_with_timeout(), 5)
-        self.assertEqual(2, counter) # 2 = original attempt plus 1 retry
-
-    def test_connection_exception_retry_success_none_response(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add_with_none_exception(), 5)
-        self.assertEqual(2, counter) # 2 = original attempt returned None, plus 1 retry
-
     def test_connection_exception_retry_success_failed_untar(self):
         Counter._reset()
         connection_exception_retry(lambda: Counter._add_with_untar_exception(), 5)
@@ -44,25 +34,18 @@ class Counter():
     def _add():
         global counter
         counter+=1
-    def _add_with_exception():
-        global counter
-        counter+=1
-        raise requests.exceptions.ConnectionError
-    def _add_with_limited_exception():
-        global counter
-        counter+=1
-        if counter < 2:
-            raise requests.exceptions.ConnectionError
-    def _add_with_timeout():
+    # All exceptions that Requests explicitly raises inherit from
+    # requests.exceptions.RequestException so we want to make sure that raises plus one exception
+    # that inherit from it for sanity
+    def _add_with_requests_exception():
         global counter
         counter+=1
         if counter < 2:
-            raise requests.exceptions.Timeout
-    def _add_with_none_exception():
+            raise requests.exceptions.RequestException
+    def _add_with_exception():
         global counter
         counter+=1
-        if counter < 2:
-            raise requests.exceptions.ContentDecodingError
+        raise requests.exceptions.ConnectionError
     def _add_with_untar_exception():
         global counter
         counter+=1
diff --git a/test/unit/test_events.py b/test/unit/test_events.py
--- a/test/unit/test_events.py
+++ b/test/unit/test_events.py
@@ -414,6 +414,7 @@ def MockNode():
     IntegrationTestError(msg=''),
     IntegrationTestException(msg=''),
     EventBufferFull(),
+    RecordRetryException(exc=Exception('')),
     UnitTestInfo(msg=''),
 ]