Repository: dbt-labs/dbt-core

Issue:
[FEATURE] Set all macro namespace by default to `['<name_of_your_root_project>', '<macro_namespace>']`
As suggested by @jtcohen6 in [comment](https://github.com/dbt-labs/dbt-core/issues/4098#issuecomment-947678409), change this https://github.com/dbt-labs/dbt-core/blob/280e9ad9c9bf914b5bbd1aa14629a6708c921546/core/dbt/context/providers.py#L146-L147

to:

``` python
            if not search_packages and namespace in self._adapter.config.dependencies:
                search_packages = [self.config.project_name, namespace]
```

With this change, by default we prefer macro overwrites for packages (like `spark__create_table_as`) defined in the root project over the ones defined in the package. The user can change the preference behavior by defining the search order explicitly:

``` yml
# dbt_project.yml
dispatch:
  - macro_namespace: dbt
    search_order: ['<package name>, '<root project name>']
```

In the old behavior the macros for packages in the root folder are not preferred over the ones in the packages, needing the user to explicitly set the search order to:

``` yml
# dbt_project.yml
dispatch:
  - macro_namespace: dbt
    search_order: ['<root project name', '<package name>]
```

We think that the proposed change is a more sensible default.


Discussion:

Resolving patch:
diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py
--- a/core/dbt/context/providers.py
+++ b/core/dbt/context/providers.py
@@ -144,7 +144,7 @@ def dispatch(
         elif isinstance(namespace, str):
             search_packages = self._adapter.config.get_macro_search_order(namespace)
             if not search_packages and namespace in self._adapter.config.dependencies:
-                search_packages = [namespace]
+                search_packages = [self.config.project_name, namespace]
         else:
             # Not a string and not None so must be a list
             raise CompilationException(
@@ -162,10 +162,10 @@ def dispatch(
                     macro = self._namespace.get_from_package(
                         package_name, search_name
                     )
-                except CompilationException as exc:
-                    raise CompilationException(
-                        f'In dispatch: {exc.msg}',
-                    ) from exc
+                except CompilationException:
+                    # Only raise CompilationException if macro is not found in
+                    # any package
+                    macro = None
 
                 if package_name is None:
                     attempts.append(search_name)


Test patch:
diff --git a/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql b/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql
new file mode 100644
--- /dev/null
+++ b/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql
@@ -0,0 +1,3 @@
+{% macro postgres__get_columns_in_relation(relation) %}
+	{{ return('a string') }}
+{% endmacro %}
diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py
--- a/test/integration/016_macro_tests/test_macros.py
+++ b/test/integration/016_macro_tests/test_macros.py
@@ -118,6 +118,64 @@ def test_postgres_overrides(self):
         self.run_dbt()
 
 
+class TestMacroOverridePackage(DBTIntegrationTest):
+    """
+    The macro in `override-postgres-get-columns-macros` should override the
+    `get_columns_in_relation` macro by default.
+    """
+
+    @property
+    def schema(self):
+        return "test_macros_016"
+
+    @property
+    def models(self):
+        return 'override-get-columns-models'
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'macro-paths': ['override-postgres-get-columns-macros'],
+        }
+
+    @use_profile('postgres')
+    def test_postgres_overrides(self):
+        # the first time, the model doesn't exist
+        self.run_dbt()
+        self.run_dbt()
+
+
+class TestMacroNotOverridePackage(DBTIntegrationTest):
+    """
+    The macro in `override-postgres-get-columns-macros` does NOT override the
+    `get_columns_in_relation` macro because we tell dispatch to not look at the
+    postgres macros.
+    """
+
+    @property
+    def schema(self):
+        return "test_macros_016"
+
+    @property
+    def models(self):
+        return 'override-get-columns-models'
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            'macro-paths': ['override-postgres-get-columns-macros'],
+            'dispatch': [{'macro_namespace': 'dbt', 'search_order': ['dbt']}],
+        }
+
+    @use_profile('postgres')
+    def test_postgres_overrides(self):
+        # the first time, the model doesn't exist
+        self.run_dbt(expect_pass=False)
+        self.run_dbt(expect_pass=False)
+
+
 class TestDispatchMacroOverrideBuiltin(TestMacroOverrideBuiltin):
     # test the same functionality as above, but this time,
     # dbt.get_columns_in_relation will dispatch to a default__ macro