Repository: sqlfluff/sqlfluff

Issue:
[BigQuery] Jinja for loop failing to parse
A simple Jinja loop with BigQuery fails to parse using sqlfluff 0.5.2, last working for this is version 0.3.6

## Expected Behaviour
No errors

## Observed Behaviour

```
[stijnzanders@stijn]$ sqlfluff parse test.sql
Traceback (most recent call last):
  File "/home/stijnzanders/.local/bin/sqlfluff", line 8, in <module>
    sys.exit(cli())
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/sqlfluff/cli/commands.py", line 582, in parse
    for parsed_string in result:
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/sqlfluff/core/linter.py", line 1469, in parse_path
    yield self.parse_string(
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/sqlfluff/core/linter.py", line 874, in parse_string
    templated_file, templater_violations = self.templater.process(
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/sqlfluff/core/templaters/jinja.py", line 299, in process
    raw_sliced, sliced_file, out_str = self.slice_file(
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/sqlfluff/core/templaters/python.py", line 287, in slice_file
    sliced_file = list(
  File "/home/stijnzanders/.local/lib/python3.8/site-packages/sqlfluff/core/templaters/python.py", line 950, in _split_uniques_coalesce_rest
    "        Attempting Subsplit [post]: %s", sub_section
UnboundLocalError: local variable 'sub_section' referenced before assignment
```

## Steps to Reproduce

SQL File:
```
{% set event_columns = ['campaign', 'click_item'] %}

SELECT
    event_id
    {% for event_column in event_columns %}
    , {{ event_column }}
    {% endfor %}
FROM events
```

## Version
```
==== sqlfluff ====
sqlfluff:           0.5.2 python:             3.8.7
dialect:         bigquery verbosity:              2
templater:          jinja
```
## Configuration
```
[sqlfluff]
dialect = bigquery

[sqlfluff:indentation]
indented_joins = False

[sqlfluff:rules:L019]
comma_style = leading
```


Discussion:

Patch:
diff --git a/src/sqlfluff/core/templaters/python.py b/src/sqlfluff/core/templaters/python.py
index 814c427e9..eb0666ced 100644
--- a/src/sqlfluff/core/templaters/python.py
+++ b/src/sqlfluff/core/templaters/python.py
@@ -739,9 +739,6 @@ class PythonTemplater(RawTemplater):
                 # recursion.
                 if len(int_file_slice.slice_buffer) > bookmark_idx:
                     # Recurse to deal with any loops separately
-                    sub_section = int_file_slice.slice_buffer[
-                        bookmark_idx : len(int_file_slice.slice_buffer)
-                    ]
                     yield from cls._split_uniques_coalesce_rest(
                         [
                             IntermediateFileSlice(
@@ -749,7 +746,10 @@ class PythonTemplater(RawTemplater):
                                 # Slicing is easy here, we have no choice
                                 slice(starts[0], int_file_slice.source_slice.stop),
                                 slice(starts[1], int_file_slice.templated_slice.stop),
-                                sub_section,
+                                # Calculate the subsection to deal with.
+                                int_file_slice.slice_buffer[
+                                    bookmark_idx : len(int_file_slice.slice_buffer)
+                                ],
                             )
                         ],
                         raw_occs,
@@ -818,15 +818,15 @@ class PythonTemplater(RawTemplater):
                     # all as a tuple, because if we could do any better
                     # we would have done it by now.
 
-                    source_slice: Optional[slice] = None
+                    # Can we identify a meaningful portion of the patch
+                    # to recurse a split?
+                    sub_section: Optional[List[RawFileSlice]] = None
                     # If it's the start, the slicing is easy
                     if starts[1] == int_file_slice.templated_slice.stop:
-                        source_slice = slice(starts[0], raw_idx)
                         sub_section = int_file_slice.slice_buffer[:this_owu_idx]
                     # If we are AFTER the previous in the template, then it's
                     # also easy. [assuming it's not the same owu]
                     elif raw_idx > starts[0] and last_owu_idx != this_owu_idx:
-                        source_slice = slice(starts[0], raw_idx)
                         if last_owu_idx:
                             sub_section = int_file_slice.slice_buffer[
                                 last_owu_idx + 1 : this_owu_idx
@@ -836,7 +836,10 @@ class PythonTemplater(RawTemplater):
 
                     # If we succeeded in one of the above, we can also recurse
                     # and be more intelligent with the other sections.
-                    if source_slice:
+                    if sub_section:
+                        # This assertion makes MyPy happy. In this case, we
+                        # never set source_slice without also setting
+                        # subsection.
                         templater_logger.debug(
                             "        Attempting Subsplit [pre]: %s, %r",
                             sub_section,
@@ -847,7 +850,7 @@ class PythonTemplater(RawTemplater):
                                 IntermediateFileSlice(
                                     "compound",
                                     # Slicing is easy here, we have no choice
-                                    source_slice,
+                                    slice(starts[0], raw_idx),
                                     slice(starts[1], template_idx),
                                     sub_section,
                                 )
@@ -949,9 +952,7 @@ class PythonTemplater(RawTemplater):
 
             if starts[1] < stops[1] and last_owu_idx is not None:
                 # Yield the end bit
-                templater_logger.debug(
-                    "        Attempting Subsplit [post]: %s", sub_section
-                )
+                templater_logger.debug("        Attempting Subsplit [post].")
                 yield from cls._split_uniques_coalesce_rest(
                     [
                         IntermediateFileSlice(


Test patch:
diff --git a/test/core/templaters/jinja_test.py b/test/core/templaters/jinja_test.py
index 17aa89184..5aeb190c9 100644
--- a/test/core/templaters/jinja_test.py
+++ b/test/core/templaters/jinja_test.py
@@ -10,12 +10,35 @@ from sqlfluff.core import Linter, FluffConfig
 JINJA_STRING = "SELECT * FROM {% for c in blah %}{{c}}{% if not loop.last %}, {% endif %}{% endfor %} WHERE {{condition}}\n\n"
 
 
-def test__templater_jinja():
+@pytest.mark.parametrize(
+    "instr, expected_outstr",
+    [
+        (
+            JINJA_STRING,
+            "SELECT * FROM f, o, o WHERE a < 10\n\n",
+        ),
+        # Test for issue #968. This was previously raising an UnboundLocalError.
+        (
+            """
+{% set event_columns = ['campaign', 'click_item'] %}
+
+SELECT
+    event_id
+    {% for event_column in event_columns %}
+    , {{ event_column }}
+    {% endfor %}
+FROM events
+            """,
+            "\n\n\nSELECT\n    event_id\n    \n    , campaign\n    \n    , click_item\n    \nFROM events\n            ",
+        ),
+    ],
+    ids=["simple", "unboundlocal_bugfix"],
+)
+def test__templater_jinja(instr, expected_outstr):
     """Test jinja templating and the treatment of whitespace."""
     t = JinjaTemplater(override_context=dict(blah="foo", condition="a < 10"))
-    instr = JINJA_STRING
     outstr, _ = t.process(in_str=instr, config=FluffConfig())
-    assert str(outstr) == "SELECT * FROM f, o, o WHERE a < 10\n\n"
+    assert str(outstr) == expected_outstr
 
 
 def test__templater_jinja_error_variable():