Repository: sqlfluff/sqlfluff

Issue:
dbt & JinjaTracer results in passing invalid query to database (was: DBT Call statement() block causes invalid query generated)
### Search before asking

- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.


### What Happened

When using the call statement() to run a query during compile time, the query generated is garbled causing the following sql error:
```
{% call statement('variables', fetch_result=true) %}

select 1 as test;

{% endcall %}

{% set test = load_result('variables')['table'].columns.TEST.values()[0] %}
```

This results in the following error:

dbt.exceptions.DatabaseException: Database Error
  001003 (42000): SQL compilation error:
  syntax error line 1 at position 0 unexpected '0'.

The query ran looks like this when looking at the query runner history in snowflake:

```
☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺_0☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺_8☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺☺_0
```

Whereas it should show:
```
select 1 as test;
```

### Expected Behaviour

Expected that the query runs properly.

### Observed Behaviour

```
=== [dbt templater] Compiling dbt project...
=== [dbt templater] Project Compiled.
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/snowflake/connections.py", line 219, in exception_handler
    yield
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/sql/connections.py", line 70, in add_query
    cursor.execute(sql, bindings)
  File "/usr/local/lib/python3.9/site-packages/snowflake/connector/cursor.py", line 794, in execute
    Error.errorhandler_wrapper(self.connection, self, error_class, errvalue)
  File "/usr/local/lib/python3.9/site-packages/snowflake/connector/errors.py", line 273, in errorhandler_wrapper
    handed_over = Error.hand_to_other_handler(
  File "/usr/local/lib/python3.9/site-packages/snowflake/connector/errors.py", line 328, in hand_to_other_handler
    cursor.errorhandler(connection, cursor, error_class, error_value)
  File "/usr/local/lib/python3.9/site-packages/snowflake/connector/errors.py", line 207, in default_errorhandler
    raise error_class(
snowflake.connector.errors.ProgrammingError: 001003 (42000): SQL compilation error:
syntax error line 1 at position 0 unexpected '0'.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/bin/sqlfluff", line 8, in <module>
    sys.exit(cli())
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1130, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1055, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1657, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 760, in invoke
    return __callback(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/cli/commands.py", line 1008, in parse
    parsed_strings = list(
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py", line 1171, in parse_path
    yield self.parse_string(
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py", line 835, in parse_string
    rendered = self.render_string(in_str, fname, config, encoding)
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/core/linter/linter.py", line 784, in render_string
    templated_file, templater_violations = self.templater.process(
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/core/templaters/base.py", line 47, in _wrapped
    return func(self, in_str=in_str, fname=fname, config=config, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/sqlfluff_templater_dbt/templater.py", line 331, in process
    processed_result = self._unsafe_process(fname_absolute_path, in_str, config)
  File "/usr/local/lib/python3.9/site-packages/sqlfluff_templater_dbt/templater.py", line 552, in _unsafe_process
    raw_sliced, sliced_file, templated_sql = self.slice_file(
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/core/templaters/jinja.py", line 462, in slice_file
    trace = tracer.trace(append_to_templated=kwargs.pop("append_to_templated", ""))
  File "/usr/local/lib/python3.9/site-packages/sqlfluff/core/templaters/slicers/tracer.py", line 77, in trace
    trace_template_output = trace_template.render()
  File "/usr/local/lib/python3.9/site-packages/jinja2/environment.py", line 1090, in render
    self.environment.handle_exception()
  File "/usr/local/lib/python3.9/site-packages/jinja2/environment.py", line 832, in handle_exception
    reraise(*rewrite_traceback_stack(source=source))
  File "/usr/local/lib/python3.9/site-packages/jinja2/_compat.py", line 28, in reraise
    raise value.with_traceback(tb)
  File "<template>", line 16, in top-level template code
  File "/usr/local/lib/python3.9/site-packages/jinja2/sandbox.py", line 462, in call
    return __context.call(__obj, *args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/dbt/clients/jinja.py", line 321, in __call__
    return self.call_macro(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/dbt/clients/jinja.py", line 248, in call_macro
    return macro(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/jinja2/runtime.py", line 679, in _invoke
    rv = self._func(*arguments)
  File "<template>", line 10, in template
  File "/usr/local/lib/python3.9/site-packages/jinja2/sandbox.py", line 462, in call
    return __context.call(__obj, *args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/base/impl.py", line 235, in execute
    return self.connections.execute(sql=sql, auto_begin=auto_begin, fetch=fetch)
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/sql/connections.py", line 122, in execute
    _, cursor = self.add_query(sql, auto_begin)
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/snowflake/connections.py", line 458, in add_query
    connection, cursor = super().add_query(
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/sql/connections.py", line 78, in add_query
    return connection, cursor
  File "/usr/local/Cellar/python@3.9/3.9.13_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/dbt/adapters/snowflake/connections.py", line 238, in exception_handler
    raise DatabaseException(msg)
dbt.exceptions.DatabaseException: Database Error
  001003 (42000): SQL compilation error:
  syntax error line 1 at position 0 unexpected '0'.
```

### How to reproduce

Use the statement() block described here:
https://docs.getdbt.com/reference/dbt-jinja-functions/statement-blocks



### Dialect

Snowflake

### Version

1.2.0 with the dbt formatter

### Configuration

```
[sqlfluff]
# verbose is an integer (0-2) indicating the level of log output
verbose = 2
# Turn off color formatting of output
nocolor = False
# Supported dialects https://docs.sqlfluff.com/en/stable/dialects.html
# Or run 'sqlfluff dialects'
dialect = snowflake
# One of [raw|jinja|python|placeholder]
templater = dbt
# Comma separated list of rules to check, default to all
rules = all
# Comma separated list of rules to exclude, or None
exclude_rules = None
# The depth to recursively parse to (0 for unlimited)
recurse = 0
# Below controls SQLFluff output, see max_line_length for SQL output
output_line_length = 80
# Number of passes to run before admitting defeat
runaway_limit = 10
# Ignore errors by category (one or more of the following, separated by commas: lexing,linting,parsing,templating)
ignore = None
# Ignore linting errors found within sections of code coming directly from
# templated code (e.g. from within Jinja curly braces. Note that it does not
# ignore errors from literal code found within template loops.
ignore_templated_areas = True
# can either be autodetect or a valid encoding e.g. utf-8, utf-8-sig
encoding = autodetect
# Ignore inline overrides (e.g. to test if still required)
disable_noqa = False
# Comma separated list of file extensions to lint
# NB: This config will only apply in the root folder
sql_file_exts = .sql,.sql.j2,.dml,.ddl
# Allow fix to run on files, even if they contain parsing errors
# Note altering this is NOT RECOMMENDED as can corrupt SQL
fix_even_unparsable = False
# Very large files can make the parser effectively hang.
# This limit skips files over a certain character length
# and warns the user what has happened.
# Set this to 0 to disable.
large_file_skip_char_limit = 20000

[sqlfluff:indentation]
# See https://docs.sqlfluff.com/en/stable/indentation.html
indented_joins = False
indented_ctes = False
indented_using_on = True
indented_on_contents = True
template_blocks_indent = True

[sqlfluff:templater]
unwrap_wrapped_queries = True

[sqlfluff:templater:jinja]
apply_dbt_builtins = True
load_macros_from_path = macros/

[sqlfluff:templater:jinja:macros]
# Macros provided as builtins for dbt projects
dbt_ref = {% macro ref(model_ref) %}{{model_ref}}{% endmacro %}
dbt_source = {% macro source(source_name, table) %}{{source_name}}_{{table}}{% endmacro %}
dbt_config = {% macro config() %}{% for k in kwargs %}{% endfor %}{% endmacro %}
dbt_var = {% macro var(variable, default='') %}item{% endmacro %}
dbt_is_incremental = {% macro is_incremental() %}True{% endmacro %}

[sqlfluff:templater:dbt]
project_dir = ./

# Some rules can be configured directly from the config common to other rules
[sqlfluff:rules]
tab_space_size = 4
max_line_length = 120
indent_unit = space
comma_style = trailing
allow_scalar = True
single_table_references = consistent
unquoted_identifiers_policy = all

# Some rules have their own specific config
[sqlfluff:rules:L003]
hanging_indents = True

[sqlfluff:rules:L007]
operator_new_lines = after

[sqlfluff:rules:L010]
# Keywords
capitalisation_policy = lower
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L011]
# Aliasing preference for tables
aliasing = explicit

[sqlfluff:rules:L012]
# Aliasing preference for columns
aliasing = explicit

[sqlfluff:rules:L014]
# Unquoted identifiers
extended_capitalisation_policy = lower
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L016]
# Line length
ignore_comment_lines = False
ignore_comment_clauses = False

[sqlfluff:rules:L027]
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L026]
# References must be in FROM clause
# Disabled for some dialects (e.g. bigquery)
force_enable = False

[sqlfluff:rules:L028]
# References must be consistently used
# Disabled for some dialects (e.g. bigquery)
force_enable = False

[sqlfluff:rules:L029]
# Keywords should not be used as identifiers.
unquoted_identifiers_policy = aliases
quoted_identifiers_policy = none
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L030]
# Function names
extended_capitalisation_policy = lower
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L031]
# Avoid table aliases in from clauses and join conditions.
# Disabled for some dialects (e.g. bigquery)
force_enable = False

[sqlfluff:rules:L036]
wildcard_policy = single

[sqlfluff:rules:L038]
# Trailing commas
select_clause_trailing_comma = forbid

[sqlfluff:rules:L040]
# Null & Boolean Literals
capitalisation_policy = consistent
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L042]
# By default, allow subqueries in from clauses, but not join clauses
forbid_subquery_in = join

[sqlfluff:rules:L047]
# Consistent syntax to count all rows
prefer_count_1 = False
prefer_count_0 = False

[sqlfluff:rules:L051]
# Fully qualify JOIN clause
fully_qualify_join_types = inner

[sqlfluff:rules:L052]
# Semi-colon formatting approach
multiline_newline = False
require_final_semicolon = False

[sqlfluff:rules:L054]
# GROUP BY/ORDER BY column references
group_by_and_order_by_style = consistent

[sqlfluff:rules:L057]
# Special characters in identifiers
unquoted_identifiers_policy = all
quoted_identifiers_policy = all
allow_space_in_identifier = False
additional_allowed_characters = ""
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L059]
# Policy on quoted and unquoted identifiers
prefer_quoted_identifiers = False
ignore_words = None
ignore_words_regex = None
force_enable = False

[sqlfluff:rules:L062]
# Comma separated list of blocked words that should not be used
blocked_words = None
blocked_regex = None

[sqlfluff:rules:L063]
# Data Types
extended_capitalisation_policy = consistent
# Comma separated list of words to ignore for this rule
ignore_words = None
ignore_words_regex = None

[sqlfluff:rules:L064]
# Consistent usage of preferred quotes for quoted literals
preferred_quoted_literal_style = consistent
# Disabled for dialects that do not support single and double quotes for quoted literals (e.g. Postgres)
force_enable = False

[sqlfluff:rules:L066]
min_alias_length = None
max_alias_length = None
```

### Are you willing to work on and submit a PR to address the issue?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)



Discussion:
This is a curious error. Can you run `dbt compile` and post what dbt expects the compiled form of this statement to be? I worry that while a query is run at compile time, this query otherwise compiles to an empty file - and that could be causing issues.
dbt compile doesn't output the call statement blocks since they're interpreted at runtime; however, we can see the output ran on the snowflake query history.

Source test.sql
```
{% call statement('variables', fetch_result=true) %}

    select 1

{% endcall %}

with source (
    select 1
)

select * from source
```

Compiled output of test.sql
```


with source (
    select 1
)

select * from source
```
The dbt [documentation](https://docs.getdbt.com/reference/dbt-jinja-functions/statement-blocks) mentions re: `statement()`:

>Volatile API
>While the statement and load_result setup works for now, we intend to improve this interface in the future. If you have questions or suggestions, please let us know in GitHub or on Slack.

So this might be a relatively lower priority issue. IIUC, it may also be dbt specific (not affecting the `jinja` templater).


I did some preliminary investigation. IIUC, SQLFluff's `JinjaTracer` should treat this:
```
{% call statement('variables', fetch_result=true) %}

select 1 as test;

{% endcall %}
```

like this:
```
{{ statement('variables', fetch_result=true) }}
```

In both cases, whatever `statement()` returns is passed through to the template output. I think this will be pretty straightforward, other than the usual trickiness of working on this complex area of the code.

Patch:
diff --git a/src/sqlfluff/core/templaters/slicers/tracer.py b/src/sqlfluff/core/templaters/slicers/tracer.py
--- a/src/sqlfluff/core/templaters/slicers/tracer.py
+++ b/src/sqlfluff/core/templaters/slicers/tracer.py
@@ -206,7 +206,8 @@ def __init__(self, raw_str: str, env: Environment):
 
         # Internal bookkeeping
         self.slice_id: int = 0
-        self.inside_set_or_macro: bool = False  # {% set %} or {% macro %}
+        # {% set %} or {% macro %} or {% call %}
+        self.inside_set_macro_or_call: bool = False
         self.inside_block = False  # {% block %}
         self.stack: List[int] = []
         self.idx_raw: int = 0
@@ -236,12 +237,18 @@ def slice_info_for_literal(self, length, prefix="") -> RawSliceInfo:
             unique_alternate_id, alternate_code, inside_block=self.inside_block
         )
 
-    def update_inside_set_or_macro_or_block(
-        self, block_type: str, trimmed_parts: List[str]
-    ) -> None:
-        """Based on block tag, update whether we're in a set/macro section."""
+    def update_inside_set_call_macro_or_block(
+        self,
+        block_type: str,
+        trimmed_parts: List[str],
+        m_open: Optional[regex.Match],
+        m_close: Optional[regex.Match],
+        tag_contents: List[str],
+    ) -> Optional[RawSliceInfo]:
+        """Based on block tag, update whether in a set/call/macro/block section."""
         if block_type == "block_start" and trimmed_parts[0] in (
             "block",
+            "call",
             "macro",
             "set",
         ):
@@ -274,16 +281,22 @@ def update_inside_set_or_macro_or_block(
                     if trimmed_parts[0] == "block":
                         self.inside_block = True
                     else:
-                        self.inside_set_or_macro = True
+                        result = None
+                        if trimmed_parts[0] == "call":
+                            assert m_open and m_close
+                            result = self.track_call(m_open, m_close, tag_contents)
+                        self.inside_set_macro_or_call = True
+                        return result
                 else:
                     raise  # pragma: no cover
         elif block_type == "block_end":
-            if trimmed_parts[0] in ("endmacro", "endset"):
-                # Exiting a set or macro.
-                self.inside_set_or_macro = False
+            if trimmed_parts[0] in ("endcall", "endmacro", "endset"):
+                # Exiting a set or macro or block.
+                self.inside_set_macro_or_call = False
             elif trimmed_parts[0] == "endblock":
                 # Exiting a {% block %} block.
                 self.inside_block = False
+        return None
 
     def make_raw_slice_info(
         self,
@@ -292,7 +305,7 @@ def make_raw_slice_info(
         inside_block: bool = False,
     ) -> RawSliceInfo:
         """Create RawSliceInfo as given, or "empty" if in set/macro block."""
-        if not self.inside_set_or_macro:
+        if not self.inside_set_macro_or_call:
             return RawSliceInfo(unique_alternate_id, alternate_code, [], inside_block)
         else:
             return RawSliceInfo(None, None, [], False)
@@ -355,6 +368,8 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:
             # raw_end and raw_begin behave a little differently in
             # that the whole tag shows up in one go rather than getting
             # parts of the tag at a time.
+            m_open = None
+            m_close = None
             if elem_type.endswith("_end") or elem_type == "raw_begin":
                 block_type = self.block_types[elem_type]
                 block_subtype = None
@@ -376,7 +391,11 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:
                         raw_slice_info = self.track_templated(
                             m_open, m_close, tag_contents
                         )
-                self.update_inside_set_or_macro_or_block(block_type, tag_contents)
+                raw_slice_info_temp = self.update_inside_set_call_macro_or_block(
+                    block_type, tag_contents, m_open, m_close, tag_contents
+                )
+                if raw_slice_info_temp:
+                    raw_slice_info = raw_slice_info_temp
                 m_strip_right = regex.search(
                     r"\s+$", raw, regex.MULTILINE | regex.DOTALL
                 )
@@ -428,6 +447,7 @@ def analyze(self, make_template: Callable[[str], Template]) -> JinjaTracer:
                     slice_idx = len(self.raw_sliced) - 1
                     self.idx_raw += len(str_buff)
                 if block_type.startswith("block"):
+                    self.track_block_start(block_type, tag_contents[0])
                     self.track_block_end(block_type, tag_contents[0])
                     self.update_next_slice_indices(
                         slice_idx, block_type, tag_contents[0]
@@ -457,6 +477,21 @@ def track_templated(
         )
         return self.make_raw_slice_info(unique_alternate_id, alternate_code)
 
+    def track_call(
+        self, m_open: regex.Match, m_close: regex.Match, tag_contents: List[str]
+    ):
+        """Set up tracking for "{% call ... %}"."""
+        unique_alternate_id = self.next_slice_id()
+        open_ = m_open.group(1)
+        close_ = m_close.group(1)
+        # Here, we still need to evaluate the original tag contents, e.g. in
+        # case it has intentional side effects, but also return a slice ID
+        # for tracking.
+        alternate_code = (
+            f"\0{unique_alternate_id} {open_} " f"{''.join(tag_contents)} {close_}"
+        )
+        return self.make_raw_slice_info(unique_alternate_id, alternate_code)
+
     def track_literal(self, raw: str, block_idx: int) -> None:
         """Set up tracking for a Jinja literal."""
         self.raw_sliced.append(
@@ -517,6 +552,20 @@ def extract_tag_contents(
             trimmed_parts = trimmed_content.split()
         return trimmed_parts
 
+    def track_block_start(self, block_type: str, tag_name: str) -> None:
+        """On starting a 'call' block, set slice_type to "templated"."""
+        if block_type == "block_start" and tag_name == "call":
+            # Replace RawSliceInfo for this slice with one that has block_type
+            # "templated".
+            old_raw_file_slice = self.raw_sliced[-1]
+            self.raw_sliced[-1] = old_raw_file_slice._replace(slice_type="templated")
+
+            # Move existing raw_slice_info entry since it's keyed by RawFileSlice.
+            self.raw_slice_info[self.raw_sliced[-1]] = self.raw_slice_info[
+                old_raw_file_slice
+            ]
+            del self.raw_slice_info[old_raw_file_slice]
+
     def track_block_end(self, block_type: str, tag_name: str) -> None:
         """On ending a 'for' or 'if' block, set up tracking."""
         if block_type == "block_end" and tag_name in (
@@ -553,7 +602,7 @@ def update_next_slice_indices(
             "endfor",
             "endif",
         ):
-            if not self.inside_set_or_macro:
+            if not self.inside_set_macro_or_call:
                 # Record potential forward jump over this block.
                 self.raw_slice_info[
                     self.raw_sliced[self.stack[-1]]


Test patch:
diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py
--- a/test/core/templaters/jinja_test.py
+++ b/test/core/templaters/jinja_test.py
@@ -697,6 +697,14 @@ def test__templater_jinja_slice_template(test, result):
     ] == result
 
 
+def _statement(*args, **kwargs):
+    return "_statement"
+
+
+def _load_result(*args, **kwargs):
+    return "_load_result"
+
+
 @pytest.mark.parametrize(
     "raw_file,override_context,result",
     [
@@ -1118,6 +1126,32 @@ def test__templater_jinja_slice_template(test, result):
                 ("literal", slice(131, 132, None), slice(88, 89, None)),
             ],
         ),
+        (
+            """{{ statement('variables', fetch_result=true) }}
+""",
+            dict(
+                statement=_statement,
+                load_result=_load_result,
+            ),
+            [
+                ("templated", slice(0, 47, None), slice(0, 10, None)),
+                ("literal", slice(47, 48, None), slice(10, 11, None)),
+            ],
+        ),
+        (
+            """{% call statement('variables', fetch_result=true) %}select 1 as test{% endcall %}
+""",
+            dict(
+                statement=_statement,
+                load_result=_load_result,
+            ),
+            [
+                ("templated", slice(0, 52, None), slice(0, 10, None)),
+                ("literal", slice(52, 68, None), slice(10, 10, None)),
+                ("block_end", slice(68, 81, None), slice(10, 10, None)),
+                ("literal", slice(81, 82, None), slice(10, 11, None)),
+            ],
+        ),
     ],
 )
 def test__templater_jinja_slice_file(raw_file, override_context, result, caplog):