Repository: dbt-labs/dbt-core

Issue:
RPC error using `set`
## Issue
The jinja block parser does not know how to handle blocks that have no `end` tag.

`set` is obvious, `do` is also a problem. We should make sure there aren't any others.

### Issue description
In general terms, please describe the issue. What command did you run?

Running commands against `dbt rpc` that have a top-level `set` that does not use the full-block form triggers an error.

### Results
What happened? What did you expect to happen?
Expected: A result
Got: An error

### System information
dbt version: `dev/wilt-chamberlain` (4dd80567e15)
The operating system you're running on: macOS

The python version you're using (probably the output of `python --version`):
python 3.6.5

### Steps to reproduce
1) Run the rpc server with `dbt rpc`
2) Make a query that uses `{% set x = 1 %}` or similar (not a full `set` block), such as `'{% set x = 1 %}select {{ x }} as id'`
3) Get an error like `'Compilation Error\n  unexpected EOF, expected "endset", got "select {{ x }} as id"'`

sample request:
```
{
  'jsonrpc': '2.0',
  'method': 'run',
  'params': {
    'sql': 'eyUgc2V0IHggPSAxICV9c2VsZWN0IHt7IHggfX0gYXMgaWQ=',
    'name': 'foo',
  },
  'id': 1
}
```

That `sql` is just `base64.encodebytes(b'{% set x = 1 %}select {{ x }} as id')`



Discussion:

Resolving patch:
diff --git a/core/dbt/clients/_jinja_blocks.py b/core/dbt/clients/_jinja_blocks.py
--- a/core/dbt/clients/_jinja_blocks.py
+++ b/core/dbt/clients/_jinja_blocks.py
@@ -16,11 +16,12 @@ def __init__(self, contents):
 
 
 class BlockTag(object):
-    def __init__(self, block_type_name, block_name, contents=None, **kw):
+    def __init__(self, block_type_name, block_name, contents=None,
+                 full_block=None, **kw):
         self.block_type_name = block_type_name
         self.block_name = block_name
         self.contents = contents
-        self.full_block = None
+        self.full_block = full_block
 
     def __str__(self):
         return 'BlockTag({!r}, {!r})'.format(self.block_type_name,
@@ -59,7 +60,7 @@ def end_pat(self):
     r'(?:\s+(?P<block_name>({})))?'.format(_NAME_PATTERN),
 )))
 
-TAG_CLOSE_PATTERN = regex(r'(?:\-\%\}\s*|\%\})')
+TAG_CLOSE_PATTERN = regex(r'(?:(?P<tag_close>(\-\%\}\s*|\%\})))')
 # if you do {% materialization foo, adapter="myadapter' %} and end up with
 # mismatched quotes this will still match, but jinja will fail somewhere
 # since the adapter= argument has to be an adapter name, and none have quotes
@@ -103,6 +104,13 @@ def end_pat(self):
 )
 
 
+NON_STRING_DO_BLOCK_MEMBER_PATTERN = regex(
+    # anything, followed by a quote, paren, or a tag end
+    r'''(.*?)'''
+    r'''((?P<quote>(['"]))|(?P<open>(\())|(?P<close>(\))))'''
+)
+
+
 class BlockIterator(object):
     def __init__(self, data):
         self.data = data
@@ -134,9 +142,7 @@ def _match(self, pattern):
     def expect_comment_end(self):
         """Expect a comment end and return the match object.
         """
-        match = self._match(COMMENT_END_PATTERN)
-        if match is None:
-            dbt.exceptions.raise_compiler_error('unexpected EOF, expected #}')
+        match = self._expect_match('#}', COMMENT_END_PATTERN)
         self.advance(match.end())
 
     def expect_raw_end(self):
@@ -195,7 +201,8 @@ def handle_block(self, match, block_start=None):
         while True:
             match = self._expect_match(
                 '"{}"'.format(found.end_block_type_name),
-                found.end_pat(), COMMENT_START_PATTERN, RAW_START_PATTERN
+                found.end_pat(), COMMENT_START_PATTERN, RAW_START_PATTERN,
+                regex('''(?P<quote>(['"]))''')
             )
             groups = match.groupdict()
             if groups.get('endblock') is not None:
@@ -207,6 +214,10 @@ def handle_block(self, match, block_start=None):
                 self.expect_comment_end()
             elif groups.get('raw_start') is not None:
                 self.expect_raw_end()
+            elif groups.get('quote') is not None:
+                self.rewind()
+                match = self._expect_match('any string', STRING_PATTERN)
+                self.advance(match.end())
             else:
                 raise dbt.exceptions.InternalException(
                     'unhandled regex in handle_block, no match: {}'
@@ -226,10 +237,49 @@ def handle_block(self, match, block_start=None):
     def handle_materialization(self, match):
         self._expect_match('materialization args',
                            MATERIALIZATION_ARGS_PATTERN)
-        self._expect_match('%}', TAG_CLOSE_PATTERN)
+        endtag = self._expect_match('%}', TAG_CLOSE_PATTERN)
+        self.advance(endtag.end())
         # handle the block we started with!
         self.blocks.append(self.handle_block(match))
 
+    def handle_do(self, match, expect_block):
+        if expect_block:
+            # we might be wrong to expect a block ({% do (...) %}, for example)
+            # so see if there's more data before the tag closes. if there
+            # isn't, we expect a block.
+            close_match = self._expect_match('%}', TAG_CLOSE_PATTERN)
+            unprocessed = self.data[match.end():close_match.start()].strip()
+            expect_block = not unprocessed
+
+        if expect_block:
+            # if we're here, expect_block is True and we must have set
+            # close_match
+            self.advance(close_match.end())
+            block = self.handle_block(match)
+        else:
+            # we have a do-statement like {% do thing() %}, so no {% enddo %}
+            # also, we don't want to advance to the end of the match, as it
+            # might be inside a string or something! So go back and figure out
+            self._process_rval_components()
+            block = BlockTag('do', None,
+                             full_block=self.data[match.start():self.pos])
+        self.blocks.append(block)
+
+    def handle_set(self, match):
+        equal_or_close = self._expect_match('%} or =',
+                                            TAG_CLOSE_PATTERN, regex(r'='))
+        self.advance(equal_or_close.end())
+        if equal_or_close.groupdict().get('tag_close') is None:
+            # it's an equals sign, must be like {% set x = 1 %}
+            self._process_rval_components()
+            # watch out, order matters here on python 2
+            block = BlockTag(full_block=self.data[match.start():self.pos],
+                             **match.groupdict())
+        else:
+            # it's a tag close, must be like {% set x %}...{% endset %}
+            block = self.handle_block(match)
+        self.blocks.append(block)
+
     def find_block(self):
         open_block = (
             r'(?:\s*\{\%\-|\{\%)\s*'
@@ -251,21 +301,39 @@ def find_block(self):
         # comments are easy
         if matchgroups.get('comment_start') is not None:
             start = match.start()
+            self.advance(match.end())
             self.expect_comment_end()
             self.blocks.append(BlockData(self.data[start:self.pos]))
             return True
 
-        if matchgroups.get('block_type_name') == 'raw':
+        block_type_name = matchgroups.get('block_type_name')
+
+        if block_type_name == 'raw':
             start = match.start()
             self.expect_raw_end()
             self.blocks.append(BlockData(self.data[start:self.pos]))
             return True
 
-        if matchgroups.get('block_type_name') == 'materialization':
+        if block_type_name == 'materialization':
             self.advance(match.end())
             self.handle_materialization(match)
             return True
 
+        if block_type_name == 'do':
+            # if there is a "block_name" in the match groups, we don't expect a
+            # block as the "block name" is actually part of the do-statement.
+            # we need to do this to handle the (weird and probably wrong!) case
+            # of a do-statement that is only a single identifier - techincally
+            # allowed in jinja. (for example, {% do thing %})
+            expect_block = matchgroups.get('block_name') is None
+            self.handle_do(match, expect_block=expect_block)
+            return True
+
+        if block_type_name == 'set':
+            self.advance(match.end())
+            self.handle_set(match)
+            return True
+
         # we're somewhere like this {% block_type_name block_type
         # we've either got arguments, a close of tag (%}), or bad input.
         # we've handled materializations already (they're weird!)
@@ -284,6 +352,42 @@ def find_block(self):
         self.blocks.append(self.handle_block(match))
         return True
 
+    def _process_rval_components(self):
+        """This is suspiciously similar to _process_macro_default_arg, probably
+        want to figure out how to merge the two.
+
+        Process the rval of an assignment statement or a do-block
+        """
+        while True:
+            match = self._expect_match(
+                'do block component',
+                # you could have a string, though that would be weird
+                STRING_PATTERN,
+                # a quote or an open/close parenthesis
+                NON_STRING_DO_BLOCK_MEMBER_PATTERN,
+                # a tag close
+                TAG_CLOSE_PATTERN
+            )
+            matchgroups = match.groupdict()
+            self.advance(match.end())
+            if matchgroups.get('string') is not None:
+                continue
+            elif matchgroups.get('quote') is not None:
+                self.rewind()
+                # now look for a string
+                match = self._expect_match('any string', STRING_PATTERN)
+                self.advance(match.end())
+            elif matchgroups.get('open'):
+                self._parenthesis_stack.append(True)
+            elif matchgroups.get('close'):
+                self._parenthesis_stack.pop()
+            elif matchgroups.get('tag_close'):
+                if self._parenthesis_stack:
+                    msg = ('Found "%}", expected ")"')
+                    dbt.exceptions.raise_compiler_error(msg)
+                return
+            # else whitespace
+
     def _process_macro_default_arg(self):
         """Handle the bit after an '=' in a macro default argument. This is
         probably the trickiest thing. The goal here is to accept all strings
@@ -297,7 +401,7 @@ def _process_macro_default_arg(self):
                 'macro argument',
                 # you could have a string
                 STRING_PATTERN,
-                # a quote, a comma, or a close parenthesis
+                # a quote, a comma, or a open/close parenthesis
                 NON_STRING_MACRO_ARGS_PATTERN,
                 # we want to "match", not "search"
                 method='match'
@@ -311,20 +415,17 @@ def _process_macro_default_arg(self):
                 # we got a bunch of data and then a string opening value.
                 # put the quote back on the menu
                 self.rewind()
-                # if we only got a single quote mark and didn't hit a string
-                # at all, this file has an unclosed quote. Fail accordingly.
-                if match.end() - match.start() == 1:
-                    msg = (
-                        'Unclosed quotation mark at position {}. Context:\n{}'
-                        .format(self.pos, self.data[self.pos-20:self.pos+20])
-                    )
-                    dbt.exceptions.raise_compiler_error(msg)
+                # now look for a string
+                match = self._expect_match('any string', STRING_PATTERN)
+                self.advance(match.end())
             elif matchgroups.get('comma') is not None:
                 # small hack: if we hit a comma and there is one parenthesis
                 # left, return to look for a new name. otherwise we're still
                 # looking for the parameter close.
                 if len(self._parenthesis_stack) == 1:
                     return
+            elif matchgroups.get('open'):
+                self._parenthesis_stack.append(True)
             elif matchgroups.get('close'):
                 self._parenthesis_stack.pop()
             else:


Test patch:
diff --git a/test/unit/test_jinja.py b/test/unit/test_jinja.py
--- a/test/unit/test_jinja.py
+++ b/test/unit/test_jinja.py
@@ -161,6 +161,116 @@ def test_comment_only(self):
         blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
         self.assertEqual(len(blocks), 0)
 
+    def test_comment_block_self_closing(self):
+        # test the case where a comment start looks a lot like it closes itself
+        # (but it doesn't in jinja!)
+        body = '{#} {% myblock foo %} {#}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 0)
+
+    def test_embedded_self_closing_comment_block(self):
+        body = '{% myblock foo %} {#}{% endmyblock %} {#}{% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 1)
+        self.assertEqual(blocks[0].full_block, body)
+        self.assertEqual(blocks[0].contents, ' {#}{% endmyblock %} {#}')
+
+    def test_set_statement(self):
+        body = '{% set x = 1 %}{% myblock foo %}hi{% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 2)
+        self.assertEqual(blocks[0].full_block, '{% set x = 1 %}')
+        self.assertEqual(blocks[1].full_block, '{% myblock foo %}hi{% endmyblock %}')
+
+    def test_set_block(self):
+        body = '{% set x %}1{% endset %}{% myblock foo %}hi{% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 2)
+        self.assertEqual(blocks[0].contents, '1')
+        self.assertEqual(blocks[0].block_type_name, 'set')
+        self.assertEqual(blocks[0].block_name, 'x')
+        self.assertEqual(blocks[1].full_block, '{% myblock foo %}hi{% endmyblock %}')
+
+    def test_crazy_set_statement(self):
+        body = '{% set x = (thing("{% myblock foo %}")) %}{% otherblock bar %}x{% endotherblock %}{% set y = otherthing("{% myblock foo %}") %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 3)
+        self.assertEqual(blocks[0].full_block, '{% set x = (thing("{% myblock foo %}")) %}')
+        self.assertEqual(blocks[0].block_type_name, 'set')
+        self.assertEqual(blocks[1].full_block, '{% otherblock bar %}x{% endotherblock %}')
+        self.assertEqual(blocks[1].block_type_name, 'otherblock')
+        self.assertEqual(blocks[2].full_block, '{% set y = otherthing("{% myblock foo %}") %}')
+        self.assertEqual(blocks[2].block_type_name, 'set')
+
+    def test_do_statement(self):
+        body = '{% do thing.update() %}{% myblock foo %}hi{% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 2)
+        self.assertEqual(blocks[0].full_block, '{% do thing.update() %}')
+        self.assertEqual(blocks[1].full_block, '{% myblock foo %}hi{% endmyblock %}')
+
+    def test_deceptive_do_statement(self):
+        body = '{% do thing %}{% myblock foo %}hi{% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 2)
+        self.assertEqual(blocks[0].full_block, '{% do thing %}')
+        self.assertEqual(blocks[1].full_block, '{% myblock foo %}hi{% endmyblock %}')
+
+    def test_do_block(self):
+        body = '{% do %}thing.update(){% enddo %}{% myblock foo %}hi{% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 2)
+        self.assertEqual(blocks[0].contents, 'thing.update()')
+        self.assertEqual(blocks[0].block_type_name, 'do')
+        self.assertEqual(blocks[1].full_block, '{% myblock foo %}hi{% endmyblock %}')
+
+    def test_crazy_do_statement(self):
+        body = '{% do (thing("{% myblock foo %}")) %}{% otherblock bar %}x{% endotherblock %}{% do otherthing("{% myblock foo %}") %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 3)
+        self.assertEqual(blocks[0].full_block, '{% do (thing("{% myblock foo %}")) %}')
+        self.assertEqual(blocks[0].block_type_name, 'do')
+        self.assertEqual(blocks[1].full_block, '{% otherblock bar %}x{% endotherblock %}')
+        self.assertEqual(blocks[1].block_type_name, 'otherblock')
+        self.assertEqual(blocks[2].full_block, '{% do otherthing("{% myblock foo %}") %}')
+        self.assertEqual(blocks[2].block_type_name, 'do')
+
+    def test_awful_jinja(self):
+        all_blocks = extract_toplevel_blocks(if_you_do_this_you_are_awful)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 4)
+        self.assertEqual(blocks[0].block_type_name, 'do')
+        self.assertEqual(blocks[0].full_block, '''{% do\n    set('foo="bar"')\n%}''')
+        self.assertEqual(blocks[1].block_type_name, 'set')
+        self.assertEqual(blocks[1].full_block, '''{% set x = ("100" + "hello'" + '%}') %}''')
+        self.assertEqual(blocks[2].block_type_name, 'archive')
+        self.assertEqual(blocks[2].contents, '\n    '.join([
+            '''{% set x = ("{% endarchive %}" + (40 * '%})')) %}''',
+            '{# {% endarchive %} #}',
+            '{% embedded %}',
+            '    some block data right here',
+            '{% endembedded %}'
+        ]))
+        self.assertEqual(blocks[3].block_type_name, 'materialization')
+        self.assertEqual(blocks[3].contents, '\nhi\n')
+
+    def test_quoted_endblock_within_block(self):
+        body = '{% myblock something -%}  {% set x = ("{% endmyblock %}") %}  {% endmyblock %}'
+        all_blocks = extract_toplevel_blocks(body)
+        blocks = [b for b in all_blocks if b.block_type_name != '__dbt__data']
+        self.assertEqual(len(blocks), 1)
+        self.assertEqual(blocks[0].block_type_name, 'myblock')
+        self.assertEqual(blocks[0].contents, '{% set x = ("{% endmyblock %}") %}  ')
+
 bar_block = '''{% mytype bar %}
 {# a comment
     that inside it has
@@ -188,3 +298,30 @@ def test_comment_only(self):
 {% mytype foo %} some stuff {% endmytype %}
 
 '''+bar_block+x_block
+
+
+if_you_do_this_you_are_awful = '''
+{#} here is a comment with a block inside {% block x %} asdf {% endblock %} {#}
+{% do
+    set('foo="bar"')
+%}
+{% set x = ("100" + "hello'" + '%}') %}
+{% archive something -%}
+    {% set x = ("{% endarchive %}" + (40 * '%})')) %}
+    {# {% endarchive %} #}
+    {% embedded %}
+        some block data right here
+    {% endembedded %}
+{%- endarchive %}
+
+{% raw %}
+    {% set x = SYNTAX ERROR}
+{% endraw %}
+
+
+{% materialization whatever, adapter='thing' %}
+hi
+{% endmaterialization %}
+'''
+
+