Repository: tree-sitter/tree-sitter-html (language: python)

Issue:
Parser failure on unclosed <head> inside unclosed <html>
Hello, I ran into this issue with difftastic and managed to trace it back to this parser. After narrowing down the HTML that was failing to parse, I managed this:

```html
<!doctype html><html><head>
```

Essentially, instead of being labelled as an implicitly closed element inside an implicitly closed element, it's labelled as an error with two start tags.

This feels undesirable, considering how a missing `</head>`, `</body>`, or `</html>` will cause the entire document (or half of it) to be enclosed in an error node, which breaks the parsing of the individual parts.


Discussion / hints:
> implicitly closed element

i guess youre confusing this with `<img>` and `<br>` and `<hr>` and ...

in other words, youre looking for a fault-tolerant html parser as used by web browsers
imo this is out of scope for tree-sitter-html

see also [How do browsers deal with malformed HTML?](https://stackoverflow.com/questions/8359532/how-do-browsers-deal-with-malformed-html)
This isn't malformed HTML; it's listed explicitly in the spec:
* https://html.spec.whatwg.org/multipage/semantics.html#the-html-element
* https://html.spec.whatwg.org/multipage/semantics.html#the-head-element
* https://html.spec.whatwg.org/multipage/sections.html#the-body-element

The end tags for `html`, `head`, and `body` can all be omitted in valid HTML.
wow, never knew that. thanks for the links

Fix patch:
diff --git a/package.json b/package.json
index e8819be..1de7d86 100644
--- a/package.json
+++ b/package.json
@@ -37,7 +37,7 @@
   },
   "tree-sitter": [
     {
-      "scope": "text.html.basic",
+      "scope": "source.html",
       "file-types": [
         "html"
       ],
@@ -47,6 +47,9 @@
       ],
       "injections": [
         "queries/injections.scm"
+      ],
+      "external-files": [
+        "src/tag.h"
       ]
     }
   ]
diff --git a/src/scanner.c b/src/scanner.c
index 9ec2915..e1a3f3d 100644
--- a/src/scanner.c
+++ b/src/scanner.c
@@ -11,7 +11,10 @@ enum TokenType {
     SELF_CLOSING_TAG_DELIMITER,
     IMPLICIT_END_TAG,
     RAW_TEXT,
-    COMMENT
+    COMMENT,
+    OMITTED_HTML_END_TAG,
+    OMITTED_HEAD_END_TAG,
+    OMITTED_BODY_END_TAG,
 };
 
 typedef struct {
@@ -26,92 +29,94 @@ typedef struct {
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 
-#define VEC_RESIZE(vec, _cap)                                                  \
-    if ((_cap) > (vec).cap && (_cap) > 0) {                                    \
-        void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0]));       \
-        assert(tmp != NULL);                                                   \
-        (vec).data = tmp;                                                      \
-        (vec).cap = (_cap);                                                    \
+#define VEC_RESIZE(vec, _cap)                                                                                          \
+    if ((_cap) > (vec).cap && (_cap) > 0) {                                                                            \
+        void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0]));                                               \
+        assert(tmp != NULL);                                                                                           \
+        (vec).data = tmp;                                                                                              \
+        (vec).cap = (_cap);                                                                                            \
     }
 
-#define VEC_GROW(vec, _cap)                                                    \
-    if ((vec).cap < (_cap)) {                                                  \
-        VEC_RESIZE((vec), (_cap));                                             \
+#define VEC_GROW(vec, _cap)                                                                                            \
+    if ((vec).cap < (_cap)) {                                                                                          \
+        VEC_RESIZE((vec), (_cap));                                                                                     \
     }
 
-#define VEC_PUSH(vec, el)                                                      \
-    if ((vec).cap == (vec).len) {                                              \
-        VEC_RESIZE((vec), MAX(16, (vec).len * 2));                             \
-    }                                                                          \
+#define VEC_PUSH(vec, el)                                                                                              \
+    if ((vec).cap == (vec).len) {                                                                                      \
+        VEC_RESIZE((vec), MAX(16, (vec).len * 2));                                                                     \
+    }                                                                                                                  \
     (vec).data[(vec).len++] = (el);
 
-#define VEC_POP(vec)                                                           \
-    {                                                                          \
-        if (VEC_BACK(vec).type == CUSTOM) {                                    \
-            tag_free(&VEC_BACK(vec));                                          \
-        }                                                                      \
-        (vec).len--;                                                           \
+#define VEC_POP(vec)                                                                                                   \
+    {                                                                                                                  \
+        if (VEC_BACK(vec).type == CUSTOM) {                                                                            \
+            tag_free(&VEC_BACK(vec));                                                                                  \
+        }                                                                                                              \
+        (vec).len--;                                                                                                   \
     }
 
 #define VEC_BACK(vec) ((vec).data[(vec).len - 1])
 
-#define VEC_FREE(vec)                                                          \
-    {                                                                          \
-        if ((vec).data != NULL)                                                \
-            free((vec).data);                                                  \
-        (vec).data = NULL;                                                     \
+#define VEC_FREE(vec)                                                                                                  \
+    {                                                                                                                  \
+        if ((vec).data != NULL)                                                                                        \
+            free((vec).data);                                                                                          \
+        (vec).data = NULL;                                                                                             \
     }
 
-#define VEC_CLEAR(vec)                                                         \
-    {                                                                          \
-        for (int i = 0; i < (vec).len; i++) {                                  \
-            tag_free(&(vec).data[i]);                                          \
-        }                                                                      \
-        (vec).len = 0;                                                         \
+#define VEC_CLEAR(vec)                                                                                                 \
+    {                                                                                                                  \
+        for (int i = 0; i < (vec).len; i++) {                                                                          \
+            tag_free(&(vec).data[i]);                                                                                  \
+        }                                                                                                              \
+        (vec).len = 0;                                                                                                 \
     }
 
-#define STRING_RESIZE(vec, _cap)                                               \
-    void *tmp = realloc((vec).data, ((_cap) + 1) * sizeof((vec).data[0]));     \
-    assert(tmp != NULL);                                                       \
-    (vec).data = tmp;                                                          \
-    memset((vec).data + (vec).len, 0,                                          \
-           (((_cap) + 1) - (vec).len) * sizeof((vec).data[0]));                \
+#define STRING_RESIZE(vec, _cap)                                                                                       \
+    void *tmp = realloc((vec).data, ((_cap) + 1) * sizeof((vec).data[0]));                                             \
+    assert(tmp != NULL);                                                                                               \
+    (vec).data = tmp;                                                                                                  \
+    memset((vec).data + (vec).len, 0, (((_cap) + 1) - (vec).len) * sizeof((vec).data[0]));                             \
     (vec).cap = (_cap);
 
-#define STRING_GROW(vec, _cap)                                                 \
-    if ((vec).cap < (_cap)) {                                                  \
-        STRING_RESIZE((vec), (_cap));                                          \
+#define STRING_GROW(vec, _cap)                                                                                         \
+    if ((vec).cap < (_cap)) {                                                                                          \
+        STRING_RESIZE((vec), (_cap));                                                                                  \
     }
 
-#define STRING_PUSH(vec, el)                                                   \
-    if ((vec).cap == (vec).len) {                                              \
-        STRING_RESIZE((vec), MAX(16, (vec).len * 2));                          \
-    }                                                                          \
+#define STRING_PUSH(vec, el)                                                                                           \
+    if ((vec).cap == (vec).len) {                                                                                      \
+        STRING_RESIZE((vec), MAX(16, (vec).len * 2));                                                                  \
+    }                                                                                                                  \
     (vec).data[(vec).len++] = (el);
 
-#define STRING_INIT(vec)                                                       \
-    {                                                                          \
-        (vec).data = calloc(1, sizeof(char) * 17);                             \
-        (vec).len = 0;                                                         \
-        (vec).cap = 16;                                                        \
+#define STRING_INIT(vec)                                                                                               \
+    {                                                                                                                  \
+        (vec).data = calloc(1, sizeof(char) * 17);                                                                     \
+        (vec).len = 0;                                                                                                 \
+        (vec).cap = 16;                                                                                                \
     }
 
-#define STRING_FREE(vec)                                                       \
-    {                                                                          \
-        if ((vec).data != NULL)                                                \
-            free((vec).data);                                                  \
-        (vec).data = NULL;                                                     \
+#define STRING_FREE(vec)                                                                                               \
+    {                                                                                                                  \
+        if ((vec).data != NULL)                                                                                        \
+            free((vec).data);                                                                                          \
+        (vec).data = NULL;                                                                                             \
     }
 
-#define STRING_CLEAR(vec)                                                      \
-    {                                                                          \
-        (vec).len = 0;                                                         \
-        memset((vec).data, 0, (vec).cap * sizeof(char));                       \
+#define STRING_CLEAR(vec)                                                                                              \
+    {                                                                                                                  \
+        (vec).len = 0;                                                                                                 \
+        memset((vec).data, 0, (vec).cap * sizeof(char));                                                               \
     }
 
+static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+
+static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
 static unsigned serialize(Scanner *scanner, char *buffer) {
-    uint16_t tag_count =
-        scanner->tags.len > UINT16_MAX ? UINT16_MAX : scanner->tags.len;
+    uint16_t tag_count = scanner->tags.len > UINT16_MAX ? UINT16_MAX : scanner->tags.len;
     uint16_t serialized_tag_count = 0;
 
     unsigned size = sizeof(tag_count);
@@ -125,8 +130,7 @@ static unsigned serialize(Scanner *scanner, char *buffer) {
             if (name_length > UINT8_MAX) {
                 name_length = UINT8_MAX;
             }
-            if (size + 2 + name_length >=
-                TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
+            if (size + 2 + name_length >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
                 break;
             }
             buffer[size++] = (char)tag.type;
@@ -152,8 +156,7 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
         uint16_t tag_count = 0;
         uint16_t serialized_tag_count = 0;
 
-        memcpy(&serialized_tag_count, &buffer[size],
-               sizeof(serialized_tag_count));
+        memcpy(&serialized_tag_count, &buffer[size], sizeof(serialized_tag_count));
         size += sizeof(serialized_tag_count);
 
         memcpy(&tag_count, &buffer[size], sizeof(tag_count));
@@ -169,10 +172,8 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
                     uint16_t name_length = (uint8_t)buffer[size++];
                     tag.custom_tag_name.len = name_length;
                     tag.custom_tag_name.cap = name_length;
-                    tag.custom_tag_name.data =
-                        (char *)calloc(1, sizeof(char) * (name_length + 1));
-                    strncpy(tag.custom_tag_name.data, &buffer[size],
-                            name_length);
+                    tag.custom_tag_name.data = (char *)calloc(1, sizeof(char) * (name_length + 1));
+                    strncpy(tag.custom_tag_name.data, &buffer[size], name_length);
                     size += name_length;
                 }
                 VEC_PUSH(scanner->tags, tag);
@@ -190,10 +191,9 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
 static String scan_tag_name(TSLexer *lexer) {
     String tag_name;
     STRING_INIT(tag_name);
-    while (iswalnum(lexer->lookahead) || lexer->lookahead == '-' ||
-           lexer->lookahead == ':') {
+    while (iswalnum(lexer->lookahead) || lexer->lookahead == '-' || lexer->lookahead == ':') {
         STRING_PUSH(tag_name, towupper(lexer->lookahead));
-        lexer->advance(lexer, false);
+        advance(lexer);
     }
     return tag_name;
 }
@@ -202,11 +202,11 @@ static bool scan_comment(TSLexer *lexer) {
     if (lexer->lookahead != '-') {
         return false;
     }
-    lexer->advance(lexer, false);
+    advance(lexer);
     if (lexer->lookahead != '-') {
         return false;
     }
-    lexer->advance(lexer, false);
+    advance(lexer);
 
     unsigned dashes = 0;
     while (lexer->lookahead) {
@@ -217,14 +217,14 @@ static bool scan_comment(TSLexer *lexer) {
             case '>':
                 if (dashes >= 2) {
                     lexer->result_symbol = COMMENT;
-                    lexer->advance(lexer, false);
+                    advance(lexer);
                     lexer->mark_end(lexer);
                     return true;
                 }
             default:
                 dashes = 0;
         }
-        lexer->advance(lexer, false);
+        advance(lexer);
     }
     return false;
 }
@@ -236,8 +236,7 @@ static bool scan_raw_text(Scanner *scanner, TSLexer *lexer) {
 
     lexer->mark_end(lexer);
 
-    const char *end_delimiter =
-        VEC_BACK(scanner->tags).type == SCRIPT ? "</SCRIPT" : "</STYLE";
+    const char *end_delimiter = VEC_BACK(scanner->tags).type == SCRIPT ? "</SCRIPT" : "</STYLE";
 
     unsigned delimiter_index = 0;
     while (lexer->lookahead) {
@@ -246,10 +245,10 @@ static bool scan_raw_text(Scanner *scanner, TSLexer *lexer) {
             if (delimiter_index == strlen(end_delimiter)) {
                 break;
             }
-            lexer->advance(lexer, false);
+            advance(lexer);
         } else {
             delimiter_index = 0;
-            lexer->advance(lexer, false);
+            advance(lexer);
             lexer->mark_end(lexer);
         }
     }
@@ -264,7 +263,7 @@ static bool scan_implicit_end_tag(Scanner *scanner, TSLexer *lexer) {
     bool is_closing_tag = false;
     if (lexer->lookahead == '/') {
         is_closing_tag = true;
-        lexer->advance(lexer, false);
+        advance(lexer);
     } else {
         if (parent && is_void(parent)) {
             VEC_POP(scanner->tags);
@@ -274,7 +273,7 @@ static bool scan_implicit_end_tag(Scanner *scanner, TSLexer *lexer) {
     }
 
     String tag_name = scan_tag_name(lexer);
-    if (tag_name.len == 0) {
+    if (tag_name.len == 0 && !lexer->eof(lexer)) {
         STRING_FREE(tag_name);
         return false;
     }
@@ -283,8 +282,7 @@ static bool scan_implicit_end_tag(Scanner *scanner, TSLexer *lexer) {
 
     if (is_closing_tag) {
         // The tag correctly closes the topmost element on the stack
-        if (scanner->tags.len > 0 &&
-            tagcmp(&VEC_BACK(scanner->tags), &next_tag)) {
+        if (scanner->tags.len > 0 && tagcmp(&VEC_BACK(scanner->tags), &next_tag)) {
             STRING_FREE(tag_name);
             tag_free(&next_tag);
             return false;
@@ -301,7 +299,9 @@ static bool scan_implicit_end_tag(Scanner *scanner, TSLexer *lexer) {
                 return true;
             }
         }
-    } else if (parent && !can_contain(parent, &next_tag)) {
+    } else if (parent &&
+               (!can_contain(parent, &next_tag) ||
+                (parent->type == HTML || parent->type == HEAD || parent->type == BODY) && lexer->eof(lexer))) {
         VEC_POP(scanner->tags);
         lexer->result_symbol = IMPLICIT_END_TAG;
         STRING_FREE(tag_name);
@@ -356,9 +356,9 @@ static bool scan_end_tag_name(Scanner *scanner, TSLexer *lexer) {
 }
 
 static bool scan_self_closing_tag_delimiter(Scanner *scanner, TSLexer *lexer) {
-    lexer->advance(lexer, false);
+    advance(lexer);
     if (lexer->lookahead == '>') {
-        lexer->advance(lexer, false);
+        advance(lexer);
         if (scanner->tags.len > 0) {
             VEC_POP(scanner->tags);
             lexer->result_symbol = SELF_CLOSING_TAG_DELIMITER;
@@ -369,22 +369,24 @@ static bool scan_self_closing_tag_delimiter(Scanner *scanner, TSLexer *lexer) {
 }
 
 static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
-    if (valid_symbols[RAW_TEXT] && !valid_symbols[START_TAG_NAME] &&
-        !valid_symbols[END_TAG_NAME]) {
+    if (scanner->tags.len > 0) {
+        Tag *parent = &VEC_BACK(scanner->tags);
+    }
+    if (valid_symbols[RAW_TEXT] && !valid_symbols[START_TAG_NAME] && !valid_symbols[END_TAG_NAME]) {
         return scan_raw_text(scanner, lexer);
     }
 
     while (iswspace(lexer->lookahead)) {
-        lexer->advance(lexer, true);
+        skip(lexer);
     }
 
     switch (lexer->lookahead) {
         case '<':
             lexer->mark_end(lexer);
-            lexer->advance(lexer, false);
+            advance(lexer);
 
             if (lexer->lookahead == '!') {
-                lexer->advance(lexer, false);
+                advance(lexer);
                 return scan_comment(lexer);
             }
 
@@ -406,12 +408,9 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
             break;
 
         default:
-            if ((valid_symbols[START_TAG_NAME] ||
-                 valid_symbols[END_TAG_NAME]) &&
-                !valid_symbols[RAW_TEXT]) {
-                return valid_symbols[START_TAG_NAME]
-                           ? scan_start_tag_name(scanner, lexer)
-                           : scan_end_tag_name(scanner, lexer);
+            if ((valid_symbols[START_TAG_NAME] || valid_symbols[END_TAG_NAME]) && !valid_symbols[RAW_TEXT]) {
+                return valid_symbols[START_TAG_NAME] ? scan_start_tag_name(scanner, lexer)
+                                                     : scan_end_tag_name(scanner, lexer);
             }
     }
 
@@ -423,21 +422,17 @@ void *tree_sitter_html_external_scanner_create() {
     return scanner;
 }
 
-bool tree_sitter_html_external_scanner_scan(void *payload, TSLexer *lexer,
-                                            const bool *valid_symbols) {
+bool tree_sitter_html_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
     Scanner *scanner = (Scanner *)payload;
     return scan(scanner, lexer, valid_symbols);
 }
 
-unsigned tree_sitter_html_external_scanner_serialize(void *payload,
-                                                     char *buffer) {
+unsigned tree_sitter_html_external_scanner_serialize(void *payload, char *buffer) {
     Scanner *scanner = (Scanner *)payload;
     return serialize(scanner, buffer);
 }
 
-void tree_sitter_html_external_scanner_deserialize(void *payload,
-                                                   const char *buffer,
-                                                   unsigned length) {
+void tree_sitter_html_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
     Scanner *scanner = (Scanner *)payload;
     deserialize(scanner, buffer, length);
 }


Test patch:
diff --git a/test/corpus/main.txt b/test/corpus/main.txt
index aa2b7f5..ce0a336 100644
--- a/test/corpus/main.txt
+++ b/test/corpus/main.txt
@@ -363,3 +363,20 @@ Multiple entities in tag contents
     (entity)
     (text)
     (end_tag (tag_name))))
+
+==================================
+Omitted end tags for html and head
+==================================
+
+<!doctype html><html><head>
+
+---
+
+(document
+  (doctype)
+  (element
+    (start_tag
+      (tag_name))
+    (element
+      (start_tag
+        (tag_name)))))