Repository: zulip/zulip

Issue:
Improve check-templates error handling for unbalanced quotes within a tag
Our template linter is good at flagging incorrect HTML, and provides blocks of offending lines in some cases, but if you delete (or add an extra) `"` in just about any tag, you get a fairly unhelpful exception like this from the linter:

```
Traceback (most recent call last):
  File "tools/check-templates", line 260, in <module>
    check_our_files()
  File "tools/check-templates", line 194, in check_our_files
    check_handlebar_templates(by_lang['handlebars'], options.modified)
  File "tools/check-templates", line 203, in check_handlebar_templates
    validate(fn, check_indent=True)
  File "tools/check-templates", line 85, in validate
    s = get_html_tag(text, state.i)
  File "tools/check-templates", line 178, in get_html_tag
    raise Exception('Tag missing >')
Exception: Tag missing >
```

`get_html_tag` has the offset of the start of the tag it's trying to match, and it'd be great to be able to indicate something clear to the user about where the error is (line number + the content of that line would probably be most useful; given how the code is setup we might find we want to write a little function `extract_line(text, offset)` that returns the line number and content of the line that offset is at within the function).



Discussion:
If we don't end up merging #2859, we should still take a look at that PR for the initial approach that was used there.

@timabbott We actually have a about five places in the code where we report errors without the more detailed line/column info, and while we test-cover all the negative cases AFAIK, we don't test that the negative cases report line/column info even when they do.

So I kind of want to fix this holistically, and make sure we add test coverage for line/column failures, and also try to simplify the parser as much as possible in terms of handling errors.  Instead of the `extract_line()` approach, I think what we want here is more of an approach where helpers can raise a simpler exception and then `tokenize()`, which has all the line/column state more easily available, can re-raise an exception with more detailed info.

I want to avoid what's happening in the current draft of #2859, which is that each helper still has to do a fair amount of boilerplate to handle simple errors.
OK, that makes sense

Resolving patch:
diff --git a/tools/lib/template_parser.py b/tools/lib/template_parser.py
--- a/tools/lib/template_parser.py
+++ b/tools/lib/template_parser.py
@@ -5,8 +5,19 @@
 import re
 
 class TemplateParserException(Exception):
-    # TODO: Have callers pass in line numbers.
-    pass
+    def __init__(self, message):
+        # type: (str) -> None
+        self.message = message
+
+    def __str__(self):
+        # type: () -> str
+        return self.message
+
+class TokenizationException(Exception):
+    def __init__(self, message, line_content=None):
+        # type: (str, str) -> None
+        self.message = message
+        self.line_content = line_content
 
 class TokenizerState(object):
     def __init__(self):
@@ -72,48 +83,53 @@ def looking_at_django_end():
     tokens = []
 
     while state.i < len(text):
-        if looking_at_comment():
-            s = get_html_comment(text, state.i)
-            tag = s[4:-3]
-            kind = 'html_comment'
-        elif looking_at_html_start():
-            s = get_html_tag(text, state.i)
-            tag_parts = s[1:-1].split()
-
-            if not tag_parts:
-                raise TemplateParserException("Tag name missing")
-
-            tag = tag_parts[0]
-
-            if is_special_html_tag(s, tag):
-                kind = 'html_special'
-            elif s.endswith('/>'):
-                kind = 'html_singleton'
+        try:
+            if looking_at_comment():
+                s = get_html_comment(text, state.i)
+                tag = s[4:-3]
+                kind = 'html_comment'
+            elif looking_at_html_start():
+                s = get_html_tag(text, state.i)
+                tag_parts = s[1:-1].split()
+
+                if not tag_parts:
+                    raise TemplateParserException("Tag name missing")
+
+                tag = tag_parts[0]
+
+                if is_special_html_tag(s, tag):
+                    kind = 'html_special'
+                elif s.endswith('/>'):
+                    kind = 'html_singleton'
+                else:
+                    kind = 'html_start'
+            elif looking_at_html_end():
+                s = get_html_tag(text, state.i)
+                tag = s[2:-1]
+                kind = 'html_end'
+            elif looking_at_handlebars_start():
+                s = get_handlebars_tag(text, state.i)
+                tag = s[3:-2].split()[0]
+                kind = 'handlebars_start'
+            elif looking_at_handlebars_end():
+                s = get_handlebars_tag(text, state.i)
+                tag = s[3:-2]
+                kind = 'handlebars_end'
+            elif looking_at_django_start():
+                s = get_django_tag(text, state.i)
+                tag = s[3:-2].split()[0]
+                kind = 'django_start'
+            elif looking_at_django_end():
+                s = get_django_tag(text, state.i)
+                tag = s[6:-3]
+                kind = 'django_end'
             else:
-                kind = 'html_start'
-        elif looking_at_html_end():
-            s = get_html_tag(text, state.i)
-            tag = s[2:-1]
-            kind = 'html_end'
-        elif looking_at_handlebars_start():
-            s = get_handlebars_tag(text, state.i)
-            tag = s[3:-2].split()[0]
-            kind = 'handlebars_start'
-        elif looking_at_handlebars_end():
-            s = get_handlebars_tag(text, state.i)
-            tag = s[3:-2]
-            kind = 'handlebars_end'
-        elif looking_at_django_start():
-            s = get_django_tag(text, state.i)
-            tag = s[3:-2].split()[0]
-            kind = 'django_start'
-        elif looking_at_django_end():
-            s = get_django_tag(text, state.i)
-            tag = s[6:-3]
-            kind = 'django_end'
-        else:
-            advance(1)
-            continue
+                advance(1)
+                continue
+        except TokenizationException as e:
+            raise TemplateParserException('''%s at Line %d Col %d:"%s"''' %
+                                          (e.message, state.line, state.col,
+                                           e.line_content))
 
         token = Token(
             kind=kind,
@@ -249,7 +265,7 @@ def get_handlebars_tag(text, i):
     while end < len(text) - 1 and text[end] != '}':
         end += 1
     if text[end] != '}' or text[end+1] != '}':
-        raise TemplateParserException('Tag missing }}')
+        raise TokenizationException('Tag missing "}}"', text[i:end+2])
     s = text[i:end+2]
     return s
 
@@ -259,7 +275,7 @@ def get_django_tag(text, i):
     while end < len(text) - 1 and text[end] != '%':
         end += 1
     if text[end] != '%' or text[end+1] != '}':
-        raise TemplateParserException('Tag missing %}')
+        raise TokenizationException('Tag missing "%}"', text[i:end+2])
     s = text[i:end+2]
     return s
 
@@ -267,20 +283,31 @@ def get_html_tag(text, i):
     # type: (str, int) -> str
     quote_count = 0
     end = i + 1
-    while end < len(text) and (text[end] != '>' or quote_count % 2 != 0):
+    unclosed_end = 0
+    while end < len(text) and (text[end] != '>' or quote_count % 2 != 0 and text[end] != '<'):
         if text[end] == '"':
             quote_count += 1
+        if not unclosed_end and text[end] == '<':
+            unclosed_end = end
         end += 1
+    if quote_count % 2 != 0:
+        if unclosed_end:
+            raise TokenizationException('Unbalanced Quotes', text[i:unclosed_end])
+        else:
+            raise TokenizationException('Unbalanced Quotes', text[i:end+1])
     if end == len(text) or text[end] != '>':
-        raise TemplateParserException('Tag missing >')
+        raise TokenizationException('Tag missing ">"', text[i:end+1])
     s = text[i:end+1]
     return s
 
 def get_html_comment(text, i):
     # type: (str, int) -> str
     end = i + 7
+    unclosed_end = 0
     while end <= len(text):
         if text[end-3:end] == '-->':
             return text[i:end]
+        if not unclosed_end and text[end] == '<':
+            unclosed_end = end
         end += 1
-    raise TemplateParserException('Unclosed comment')
+    raise TokenizationException('Unclosed comment', text[i:unclosed_end])


Test patch:
diff --git a/tools/tests/test_template_parser.py b/tools/tests/test_template_parser.py
--- a/tools/tests/test_template_parser.py
+++ b/tools/tests/test_template_parser.py
@@ -112,42 +112,51 @@ def test_validate_incomplete_handlebars_tag_1(self):
         my_html = '''
             {{# foo
         '''
-        self._assert_validate_error('Tag missing }}', text=my_html)
+        self._assert_validate_error('''Tag missing "}}" at Line 2 Col 13:"{{# foo
+        "''', text=my_html)
 
     def test_validate_incomplete_handlebars_tag_2(self):
         # type: () -> None
         my_html = '''
             {{# foo }
         '''
-        self._assert_validate_error('Tag missing }}', text=my_html)
+        self._assert_validate_error('Tag missing "}}" at Line 2 Col 13:"{{# foo }\n"', text=my_html)
 
     def test_validate_incomplete_django_tag_1(self):
         # type: () -> None
         my_html = '''
             {% foo
         '''
-        self._assert_validate_error('Tag missing %}', text=my_html)
+        self._assert_validate_error('''Tag missing "%}" at Line 2 Col 13:"{% foo
+        "''', text=my_html)
 
     def test_validate_incomplete_django_tag_2(self):
         # type: () -> None
         my_html = '''
             {% foo %
         '''
-        self._assert_validate_error('Tag missing %}', text=my_html)
+        self._assert_validate_error('Tag missing "%}" at Line 2 Col 13:"{% foo %\n"', text=my_html)
 
     def test_validate_incomplete_html_tag_1(self):
         # type: () -> None
         my_html = '''
             <b
         '''
-        self._assert_validate_error('Tag missing >', text=my_html)
+        self._assert_validate_error('''Tag missing ">" at Line 2 Col 13:"<b
+        "''', text=my_html)
 
     def test_validate_incomplete_html_tag_2(self):
         # type: () -> None
         my_html = '''
             <a href="
         '''
-        self._assert_validate_error('Tag missing >', text=my_html)
+        my_html1 = '''
+            <a href=""
+        '''
+        self._assert_validate_error('''Tag missing ">" at Line 2 Col 13:"<a href=""
+        "''', text=my_html1)
+        self._assert_validate_error('''Unbalanced Quotes at Line 2 Col 13:"<a href="
+        "''', text=my_html)
 
     def test_validate_empty_html_tag(self):
         # type: () -> None