Repository: Python-Markdown/markdown (python)

Issue:
markdown.extensions.md_in_html fails with TypeError: expected string or bytes-like object
I'm running into an issue with "simple" Markdown + HTML examples crashing with a `TypeError`. For example, the following throws a `TypeError: expected string or bytes-like object` exception:

```python
markdown.markdown(
    'Hello\n<div markdown="1"></div>', 
    extensions=["markdown.extensions.md_in_html"],
)
```

Versions: `Markdown==3.3.1`, CPython 3.8.6

I haven't had time yet to investigate this further, but I wanted to get an issue opened with some failing test cases in case others are running into this as well. I suspect this has something to do with the raw HTML parser rewrite in #803.

## Steps to reproduce:

Save the following as `mderror.py`:

```python
import traceback
import markdown


def try_render(text):
    print(f"\n\n---- Rendering: {text!r}")
    try:
        html = markdown.markdown(
            text, extensions=["markdown.extensions.md_in_html"],
        )
        print(f"---- PASS: {html!r}")
    except Exception as e:
        print("---- FAILED")
        traceback.print_exc()


try_render('Hello<div markdown="1"></div>')  # Error
try_render('Hello\n<div markdown="1"></div>')  # Error
try_render('Hello\n\n<div markdown="1"></div>')  # Okay
```

Result:

```console
$ python mderror.py


---- Rendering: 'Hello<div markdown="1"></div>'
<Element 'div' at 0x110758310>
---- FAILED
Traceback (most recent call last):
  File "mderror.py", line 8, in try_render
    html = markdown.markdown(
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/core.py", line 386, in markdown
    return md.convert(text)
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/core.py", line 290, in convert
    output = pp.run(output)
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/postprocessors.py", line 73, in run
    if self.isblocklevel(html):
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/postprocessors.py", line 94, in isblocklevel
    m = re.match(r"^\<\/?([^ >]+)", html)
  File "/Users/dbader/.pyenv/versions/3.8.6/lib/python3.8/re.py", line 191, in match
    return _compile(pattern, flags).match(string)
TypeError: expected string or bytes-like object


---- Rendering: 'Hello\n<div markdown="1"></div>'
<Element 'div' at 0x110758900>
---- FAILED
Traceback (most recent call last):
  File "mderror.py", line 8, in try_render
    html = markdown.markdown(
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/core.py", line 386, in markdown
    return md.convert(text)
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/core.py", line 290, in convert
    output = pp.run(output)
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/postprocessors.py", line 73, in run
    if self.isblocklevel(html):
  File "PROJECT_ROOT/venv/lib/python3.8/site-packages/markdown/postprocessors.py", line 94, in isblocklevel
    m = re.match(r"^\<\/?([^ >]+)", html)
  File "/Users/dbader/.pyenv/versions/3.8.6/lib/python3.8/re.py", line 191, in match
    return _compile(pattern, flags).match(string)
TypeError: expected string or bytes-like object


---- Rendering: 'Hello\n\n<div markdown="1"></div>'

---- PASS: '<p>Hello</p>\n<div></div>'
```

Pull request description:
Completes pull #1043 which fixes #1040
Completes the tests in #1043 which fixes #1040

Patch:
diff --git a/markdown/extensions/md_in_html.py b/markdown/extensions/md_in_html.py
index 174224a..f635563 100644
--- a/markdown/extensions/md_in_html.py
+++ b/markdown/extensions/md_in_html.py
@@ -86,6 +86,14 @@ class HTMLExtractorExtra(HTMLExtractor):
         else:  # pragma: no cover
             return None
 
+    def at_line_start(self):
+        """At line start."""
+
+        value = super().at_line_start()
+        if not value and self.cleandoc and self.cleandoc[-1].endswith('\n'):
+            value = True
+        return value
+
     def handle_starttag(self, tag, attrs):
         if tag in block_level_tags:
             # Valueless attr (ex: `<tag checked>`) results in `[('checked', None)]`.
@@ -93,7 +101,7 @@ class HTMLExtractorExtra(HTMLExtractor):
             attrs = {key: value if value is not None else key for key, value in attrs}
             state = self.get_state(tag, attrs)
 
-            if self.inraw or (state in [None, 'off'] and not self.mdstack):
+            if self.inraw or (state in [None, 'off'] and not self.mdstack) or not self.at_line_start():
                 # fall back to default behavior
                 attrs.pop('markdown', None)
                 super().handle_starttag(tag, attrs)
@@ -111,7 +119,10 @@ class HTMLExtractorExtra(HTMLExtractor):
                 super().handle_starttag(tag, attrs)
             else:
                 text = self.get_starttag_text()
-                self.handle_data(text)
+                if self.mdstate and self.mdstate[-1] == "off":
+                    self.handle_data(self.md.htmlStash.store(text))
+                else:
+                    self.handle_data(text)
 
     def handle_endtag(self, tag):
         if tag in block_level_tags:
@@ -128,20 +139,32 @@ class HTMLExtractorExtra(HTMLExtractor):
                 if not self.mdstack:
                     # Last item in stack is closed. Stash it
                     element = self.get_element()
+                    # Get last entry to see if it ends in newlines
+                    # If it is an element, assume there is no newlines
+                    item = self.cleandoc[-1] if self.cleandoc else ''
+                    # If we only have one newline before block element, add another
+                    if not item.endswith('\n\n') and item.endswith('\n'):
+                        self.cleandoc.append('\n')
                     self.cleandoc.append(self.md.htmlStash.store(element))
                     self.cleandoc.append('\n\n')
                     self.state = []
             else:
                 # Treat orphan closing tag as a span level tag.
                 text = self.get_endtag_text(tag)
-                self.handle_data(text)
+                if self.mdstate and self.mdstate[-1] == "off":
+                    self.handle_data(self.md.htmlStash.store(text))
+                else:
+                    self.handle_data(text)
         else:
             # Span level tag
             if self.inraw:
                 super().handle_endtag(tag)
             else:
                 text = self.get_endtag_text(tag)
-                self.handle_data(text)
+                if self.mdstate and self.mdstate[-1] == "off":
+                    self.handle_data(self.md.htmlStash.store(text))
+                else:
+                    self.handle_data(text)
 
     def handle_data(self, data):
         if self.inraw or not self.mdstack:
@@ -156,7 +179,10 @@ class HTMLExtractorExtra(HTMLExtractor):
             if self.at_line_start() and is_block:
                 self.handle_data('\n' + self.md.htmlStash.store(data) + '\n\n')
             else:
-                self.handle_data(data)
+                if self.mdstate and self.mdstate[-1] == "off":
+                    self.handle_data(self.md.htmlStash.store(data))
+                else:
+                    self.handle_data(data)
 
 
 class HtmlBlockPreprocessor(Preprocessor):


Test patch:
diff --git a/tests/test_syntax/extensions/test_md_in_html.py b/tests/test_syntax/extensions/test_md_in_html.py
index 433cdd5..946e922 100644
--- a/tests/test_syntax/extensions/test_md_in_html.py
+++ b/tests/test_syntax/extensions/test_md_in_html.py
@@ -390,6 +390,166 @@ class TestMdInHTML(TestCase):
             )
         )
 
+    def test_orphan_end_tag_in_raw_html(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                <div markdown="1">
+                <div>
+                Test
+
+                </pre>
+
+                Test
+                </div>
+                </div>
+                """
+            ),
+            self.dedent(
+                """
+                <div>
+                <div>
+                Test
+
+                </pre>
+
+                Test
+                </div>
+                </div>
+                """
+            )
+        )
+
+    def test_complex_nested_case(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                <div markdown="1">
+                **test**
+                <div>
+                **test**
+                <img src=""/>
+                <code>Test</code>
+                <span>**test**</span>
+                <p>Test 2</p>
+                </div>
+                </div>
+                """
+            ),
+            self.dedent(
+                """
+                <div>
+                <p><strong>test</strong></p>
+                <div>
+                **test**
+                <img src=""/>
+                <code>Test</code>
+                <span>**test**</span>
+                <p>Test 2</p>
+                </div>
+                </div>
+                """
+            )
+        )
+
+    def test_complex_nested_case_whitespace(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                Text with space\t
+                <div markdown="1">\t
+                \t
+                 <div>
+                **test**
+                <img src=""/>
+                <code>Test</code>
+                <span>**test**</span>
+                  <div>With whitespace</div>
+                <p>Test 2</p>
+                </div>
+                **test**
+                </div>
+                """
+            ),
+            self.dedent(
+                """
+                <p>Text with space </p>
+                <div>
+                <div>
+                **test**
+                <img src=""/>
+                <code>Test</code>
+                <span>**test**</span>
+                  <div>With whitespace</div>
+                <p>Test 2</p>
+                </div>
+                <p><strong>test</strong></p>
+                </div>
+                """
+            )
+        )
+
+    def test_md1_intail_md1(self):
+        self.assertMarkdownRenders(
+            '<div markdown="1">*foo*</div><div markdown="1">*bar*</div>',
+            self.dedent(
+                """
+                <div>
+                <p><em>foo</em></p>
+                </div>
+                <div>
+                <p><em>bar</em></p>
+                </div>
+                """
+            )
+        )
+
+    def test_md1_no_blank_line_before(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                A _Markdown_ paragraph with no blank line after.
+                <div markdown="1">
+                A _Markdown_ paragraph in an HTML block with no blank line before.
+                </div>
+                """
+            ),
+            self.dedent(
+                """
+                <p>A <em>Markdown</em> paragraph with no blank line after.</p>
+                <div>
+                <p>A <em>Markdown</em> paragraph in an HTML block with no blank line before.</p>
+                </div>
+                """
+            )
+        )
+
+    def test_md1_no_line_break(self):
+        # The div here is parsed as a span-level element. Bad input equals bad output!
+        self.assertMarkdownRenders(
+            'A _Markdown_ paragraph with <div markdown="1">no _line break_.</div>',
+            '<p>A <em>Markdown</em> paragraph with <div markdown="1">no <em>line break</em>.</div></p>'
+        )
+
+    def test_md1_in_tail(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                <div></div><div markdown="1">
+                A _Markdown_ paragraph in an HTML block in tail of previous element.
+                </div>
+                """
+            ),
+            self.dedent(
+                """
+                <div></div>
+                <div>
+                <p>A <em>Markdown</em> paragraph in an HTML block in tail of previous element.</p>
+                </div>
+                """
+            )
+        )
+
     def test_md_span_paragraph(self):
         self.assertMarkdownRenders(
             '<p markdown="span">*foo*</p>',