Repository: dbt-labs/dbt-core

Issue:
RPC Server: return compiled SQL in exceptions when tasks fail during execution
## Feature

### Feature description
The RPC Server should return the compiled SQL for an `execute` task if the query fails after compilation. The RPC server exception response should include the compiled SQL to help users understand what part of their query failed.

In the current implementation, users can see the error from the database, but _not_ the query that failed. This makes it difficult to debug errant queries.


Discussion:

Resolving patch:
diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py
--- a/core/dbt/exceptions.py
+++ b/core/dbt/exceptions.py
@@ -95,6 +95,17 @@ def __str__(self, prefix="! "):
         return lines[0] + "\n" + "\n".join(
             ["  " + line for line in lines[1:]])
 
+    def data(self):
+        result = Exception.data(self)
+        if self.node is None:
+            return result
+
+        result.update({
+            'raw_sql': self.node.get('raw_sql'),
+            'compiled_sql': self.node.get('injected_sql'),
+        })
+        return result
+
 
 class RPCFailureResult(RuntimeException):
     CODE = 10002
@@ -106,13 +117,16 @@ class RPCTimeoutException(RuntimeException):
     MESSAGE = 'RPC timeout error'
 
     def __init__(self, timeout):
+        super(RPCTimeoutException, self).__init__(self.MESSAGE)
         self.timeout = timeout
 
     def data(self):
-        return {
+        result = super(RPCTimeoutException, self).data()
+        result.update({
             'timeout': self.timeout,
             'message': 'RPC timed out after {}s'.format(self.timeout),
-        }
+        })
+        return result
 
 
 class DatabaseException(RuntimeException):
diff --git a/core/dbt/node_runners.py b/core/dbt/node_runners.py
--- a/core/dbt/node_runners.py
+++ b/core/dbt/node_runners.py
@@ -526,6 +526,8 @@ def __init__(self, config, adapter, node, node_index, num_nodes):
 
     def handle_exception(self, e, ctx):
         if isinstance(e, dbt.exceptions.Exception):
+            if hasattr(e, 'node'):
+                e.node = ctx.node
             return rpc.dbt_error(e)
         elif isinstance(e, rpc.RPCException):
             return e
diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py
--- a/core/dbt/task/compile.py
+++ b/core/dbt/task/compile.py
@@ -38,7 +38,7 @@ class RemoteCompileTask(CompileTask, RemoteCallable):
 
     def __init__(self, args, config, manifest):
         super(RemoteCompileTask, self).__init__(args, config)
-        self._base_manifest = manifest
+        self._base_manifest = manifest.deepcopy(config=config)
 
     def get_runner_type(self):
         return RPCCompileRunner
@@ -82,6 +82,7 @@ def _get_exec_node(self, name, sql, macros):
                 resource_type=NodeType.Macro
             ))
 
+        self._base_manifest.macros.update(macro_overrides)
         rpc_parser = RPCCallParser(
             self.config,
             all_projects=all_projects,


Test patch:
diff --git a/test/integration/042_sources_test/macros/macro.sql b/test/integration/042_sources_test/macros/macro.sql
--- a/test/integration/042_sources_test/macros/macro.sql
+++ b/test/integration/042_sources_test/macros/macro.sql
@@ -1,5 +1,5 @@
 {% macro override_me() -%}
-	exceptions.raise_compiler_error('this is a bad macro')
+	{{ exceptions.raise_compiler_error('this is a bad macro') }}
 {%- endmacro %}
 
 {% macro happy_little_macro() -%}
diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py
--- a/test/integration/042_sources_test/test_sources.py
+++ b/test/integration/042_sources_test/test_sources.py
@@ -389,6 +389,16 @@ def assertIsErrorWithCode(self, data, code):
         self.assertEqual(error['code'], code)
         return error
 
+    def assertIsErrorWith(self, data, code, message, error_data):
+        error = self.assertIsErrorWithCode(data, code)
+        if message is not None:
+            self.assertEqual(error['message'], message)
+
+        if error_data is not None:
+            return self.assertHasErrorData(error, error_data)
+        else:
+            return error.get('data')
+
     def assertResultHasSql(self, data, raw_sql, compiled_sql=None):
         if compiled_sql is None:
             compiled_sql = raw_sql
@@ -416,7 +426,7 @@ def assertSuccessfulRunResult(self, data, raw_sql, compiled_sql=None, table=None
         self.assertResultHasTimings(result, 'compile', 'execute')
 
     @use_profile('postgres')
-    def test_compile(self):
+    def test_compile_postgres(self):
         trivial = self.query(
             'compile',
             'select 1 as id',
@@ -500,7 +510,7 @@ def test_compile(self):
         )
 
     @use_profile('postgres')
-    def test_run(self):
+    def test_run_postgres(self):
         # seed + run dbt to make models before using them!
         self.run_dbt_with_vars(['seed'])
         self.run_dbt_with_vars(['run'])
@@ -623,28 +633,24 @@ def test_run(self):
         )
 
     @use_profile('postgres')
-    def test_invalid_requests(self):
+    def test_invalid_requests_postgres(self):
         data = self.query(
             'xxxxxnotamethodxxxxx',
             'hi this is not sql'
         ).json()
-        error = self.assertIsErrorWithCode(data, -32601)
-        self.assertEqual(error['message'],  'Method not found')
+        self.assertIsErrorWith(data, -32601, 'Method not found', None)
 
         data = self.query(
             'compile',
             'select * from {{ reff("nonsource_descendant") }}',
             name='mymodel'
         ).json()
-        error = self.assertIsErrorWithCode(data, 10004)
-        self.assertEqual(error['message'], 'Compilation Error')
-        self.assertIn('data', error)
-        error_data = error['data']
-        self.assertEqual(error_data['type'], 'CompilationException')
-        self.assertEqual(
-            error_data['message'],
-            "Compilation Error in rpc mymodel (from remote system)\n  'reff' is undefined"
-        )
+        error_data = self.assertIsErrorWith(data, 10004, 'Compilation Error', {
+            'type': 'CompilationException',
+            'message': "Compilation Error in rpc mymodel (from remote system)\n  'reff' is undefined",
+            'compiled_sql': None,
+            'raw_sql': 'select * from {{ reff("nonsource_descendant") }}',
+        })
         self.assertIn('logs', error_data)
         self.assertTrue(len(error_data['logs']) > 0)
 
@@ -653,15 +659,12 @@ def test_invalid_requests(self):
             'hi this is not sql',
             name='foo'
         ).json()
-        error = self.assertIsErrorWithCode(data, 10003)
-        self.assertEqual(error['message'], 'Database Error')
-        self.assertIn('data', error)
-        error_data = error['data']
-        self.assertEqual(error_data['type'], 'DatabaseException')
-        self.assertEqual(
-            error_data['message'],
-            'Database Error\n  syntax error at or near "hi"\n  LINE 1: hi this is not sql\n          ^'
-        )
+        error_data = self.assertIsErrorWith(data, 10003, 'Database Error', {
+            'type': 'DatabaseException',
+            'message': 'Database Error in rpc foo (from remote system)\n  syntax error at or near "hi"\n  LINE 1: hi this is not sql\n          ^',
+            'compiled_sql': 'hi this is not sql',
+            'raw_sql': 'hi this is not sql',
+        })
         self.assertIn('logs', error_data)
         self.assertTrue(len(error_data['logs']) > 0)
 
@@ -670,14 +673,24 @@ def test_invalid_requests(self):
             'select {{ happy_little_macro() }}',
             name='foo',
         ).json()
-        self.assertIsErrorWithCode(macro_no_override, 10003)
-        self.assertEqual(error['message'], 'Database Error')
+        error_data = self.assertIsErrorWith(macro_no_override, 10004, 'Compilation Error', {
+            'type': 'CompilationException',
+            'raw_sql': 'select {{ happy_little_macro() }}',
+            'compiled_sql': None
+        })
+        self.assertIn('logs', error_data)
+        self.assertTrue(len(error_data['logs']) > 0)
+
+    def assertHasErrorData(self, error, expected_error_data):
         self.assertIn('data', error)
         error_data = error['data']
-        self.assertEqual(error_data['type'], 'DatabaseException')
+        for key, value in expected_error_data.items():
+            self.assertIn(key, error_data)
+            self.assertEqual(error_data[key], value)
+        return error_data
 
     @use_profile('postgres')
-    def test_timeout(self):
+    def test_timeout_postgres(self):
         data = self.query(
             'run',
             'select from pg_sleep(5)',