Repository: dbt-labs/dbt-core

Issue:
Infinite recursion when checking for modified upstream macro
Using v0.21.0b2, the [recursive logic](https://github.com/dbt-labs/dbt/blob/d3826e670f0dbe096a7b60189dd20121805572e2/core/dbt/graph/selector_methods.py#L408-L424) that checks for upstream macro modifications in `state:modified` can encounter an infinite recursion:

```
  File "/usr/local/lib/python3.8/dist-packages/dbt/graph/selector_methods.py", line 428, in check_modified
    upstream_macro_change = self.recursively_check_macros_modified(new)
  File "/usr/local/lib/python3.8/dist-packages/dbt/graph/selector_methods.py", line 421, in recursively_check_macros_modified
    return self.recursively_check_macros_modified(macro)
  File "/usr/local/lib/python3.8/dist-packages/dbt/graph/selector_methods.py", line 421, in recursively_check_macros_modified
    return self.recursively_check_macros_modified(macro)
  File "/usr/local/lib/python3.8/dist-packages/dbt/graph/selector_methods.py", line 421, in recursively_check_macros_modified
    return self.recursively_check_macros_modified(macro)
  [Previous line repeated 954 more times]
  File "/usr/local/lib/python3.8/dist-packages/dbt/graph/selector_methods.py", line 420, in recursively_check_macros_modified
    if len(macro.depends_on.macros) > 0:
RecursionError: maximum recursion depth exceeded while calling a Python object
```

We haven't been able to identify the root cause. I see at least one way that this logic can be improved, by skipping over the loop if no macros have been modified:
```python
def recursively_check_macros_modified(self, node):
        # check if there are any changes to any macros the first time
        if self.modified_macros is None:
            self.modified_macros = self._macros_modified()
        
        # no macros have been modified, skip looping
        if not self.modified_macros:
            return False

        # loop through all macros that this node depends on
        for macro_uid in node.depends_on.macros:
            # is this macro one of the modified macros?
            if macro_uid in self.modified_macros:
                return True
            # if not, and this macro depends on other macros, keep looping
            macro = self.manifest.macros[macro_uid]
            if len(macro.depends_on.macros) > 0:
                return self.recursively_check_macros_modified(macro)
            else:
                return False
        return False
```

I'd appreciate guidance from the Core team on what other conditions might cause this logic to loop infinitely. I've tried a macro that calls itself; this does raise a recursion error, but in a different part of the codebase: [`dbt.clients.jinja.track_call`](https://github.com/dbt-labs/dbt/blob/d3826e670f0dbe096a7b60189dd20121805572e2/core/dbt/clients/jinja.py#L314).


Discussion:
There are loops in the depends_on macros. Both the 'recursively_get_depends_on_macros' in core/dbt/context/macro_resolver.py and in 'recursively_gather_macro_references' in core/dbt/parser/partial.py skip the loops by checking whether the macro unique_id has already been encountered. Although you don't need to create a list of macro unique_ids for other purposes, you probably need to create one to check to avoid the loops.

Resolving patch:
diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py
--- a/core/dbt/graph/selector_methods.py
+++ b/core/dbt/graph/selector_methods.py
@@ -405,27 +405,38 @@ def _macros_modified(self) -> List[str]:
 
         return modified
 
-    def recursively_check_macros_modified(self, node):
-        # check if there are any changes in macros the first time
-        if self.modified_macros is None:
-            self.modified_macros = self._macros_modified()
-
+    def recursively_check_macros_modified(self, node, previous_macros):
         # loop through all macros that this node depends on
         for macro_uid in node.depends_on.macros:
+            # avoid infinite recursion if we've already seen this macro
+            if macro_uid in previous_macros:
+                continue
+            previous_macros.append(macro_uid)
             # is this macro one of the modified macros?
             if macro_uid in self.modified_macros:
                 return True
             # if not, and this macro depends on other macros, keep looping
-            macro = self.manifest.macros[macro_uid]
-            if len(macro.depends_on.macros) > 0:
-                return self.recursively_check_macros_modified(macro)
+            macro_node = self.manifest.macros[macro_uid]
+            if len(macro_node.depends_on.macros) > 0:
+                return self.recursively_check_macros_modified(macro_node, previous_macros)
             else:
                 return False
-        return False
+
+    def check_macros_modified(self, node):
+        # check if there are any changes in macros the first time
+        if self.modified_macros is None:
+            self.modified_macros = self._macros_modified()
+        # no macros have been modified, skip looping entirely
+        if not self.modified_macros:
+            return False
+        # recursively loop through upstream macros to see if any is modified
+        else:
+            previous_macros = []
+            return self.recursively_check_macros_modified(node, previous_macros)
 
     def check_modified(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
         different_contents = not new.same_contents(old)  # type: ignore
-        upstream_macro_change = self.recursively_check_macros_modified(new)
+        upstream_macro_change = self.check_macros_modified(new)
         return different_contents or upstream_macro_change
 
     def check_modified_body(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
@@ -457,7 +468,7 @@ def check_modified_relation(
             return False
 
     def check_modified_macros(self, _, new: SelectorTarget) -> bool:
-        return self.recursively_check_macros_modified(new)
+        return self.check_macros_modified(new)
 
     def check_new(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
         return old is None


Test patch:
diff --git a/test/integration/062_defer_state_test/macros/infinite_macros.sql b/test/integration/062_defer_state_test/macros/infinite_macros.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/062_defer_state_test/macros/infinite_macros.sql
@@ -0,0 +1,13 @@
+{# trigger infinite recursion if not handled #}
+
+{% macro my_infinitely_recursive_macro() %}
+  {{ return(adapter.dispatch('my_infinitely_recursive_macro')()) }}
+{% endmacro %}
+
+{% macro default__my_infinitely_recursive_macro() %}
+    {% if unmet_condition %}
+        {{ my_infinitely_recursive_macro() }}
+    {% else %}
+        {{ return('') }}
+    {% endif %}
+{% endmacro %}
diff --git a/test/integration/062_defer_state_test/models/view_model.sql b/test/integration/062_defer_state_test/models/view_model.sql
--- a/test/integration/062_defer_state_test/models/view_model.sql
+++ b/test/integration/062_defer_state_test/models/view_model.sql
@@ -1 +1,4 @@
 select * from {{ ref('seed') }}
+
+-- establish a macro dependency that trips infinite recursion if not handled
+-- depends on: {{ my_infinitely_recursive_macro() }}
\ No newline at end of file