Repository: matthewwithanm/python-markdownify

Issue:
Missing blank lines above headings or code blocks
Headings are supposed to be surrounded by blank lines. However,

```"Some text.\n\n\n<h4>A heading</h4>\n\n<p>\nA paragraph"```

gets converted to:

```'Some text.\n#### A heading\n\nA paragraph\n\n'```
With just one line-break above the heading.

The same is true for code-blocks, where:

```"Some text.\n<div class=\"code\"><pre>ls</pre></div><p>A paragraph</p>"```

gets converted to:

```Some text.\n```\nls\n```\n\nA paragraph\n\n```


Discussion:

Patch:
diff --git a/README.rst b/README.rst
index af87f51..34ed7e0 100644
--- a/README.rst
+++ b/README.rst
@@ -139,6 +139,11 @@ keep_inline_images_in
   that should be allowed to contain inline images, for example ``['td']``.
   Defaults to an empty list.
 
+table_infer_header
+  Controls handling of tables with no header row (as indicated by ``<thead>``
+  or ``<th>``). When set to ``True``, the first body row is used as the header row.
+  Defaults to ``False``, which leaves the header row empty.
+
 wrap, wrap_width
   If ``wrap`` is set to ``True``, all text paragraphs are wrapped at
   ``wrap_width`` characters. Defaults to ``False`` and ``80``.
diff --git a/markdownify/__init__.py b/markdownify/__init__.py
index ac53077..776e75e 100644
--- a/markdownify/__init__.py
+++ b/markdownify/__init__.py
@@ -102,6 +102,7 @@ class MarkdownConverter(object):
         strong_em_symbol = ASTERISK
         sub_symbol = ''
         sup_symbol = ''
+        table_infer_header = False
         wrap = False
         wrap_width = 80
 
@@ -123,9 +124,9 @@ class MarkdownConverter(object):
         return self.convert_soup(soup)
 
     def convert_soup(self, soup):
-        return self.process_tag(soup, convert_as_inline=False, children_only=True)
+        return self.process_tag(soup, convert_as_inline=False)
 
-    def process_tag(self, node, convert_as_inline, children_only=False):
+    def process_tag(self, node, convert_as_inline):
         text = ''
 
         # markdown headings or cells can't include
@@ -134,7 +135,7 @@ class MarkdownConverter(object):
         isCell = node.name in ['td', 'th']
         convert_children_as_inline = convert_as_inline
 
-        if not children_only and (isHeading or isCell):
+        if isHeading or isCell:
             convert_children_as_inline = True
 
         # Remove whitespace-only textnodes just before, after or
@@ -170,13 +171,18 @@ class MarkdownConverter(object):
                 newlines = '\n' * max(newlines_left, newlines_right)
                 text = text_strip + newlines + next_text_strip
 
-        if not children_only:
-            convert_fn = getattr(self, 'convert_%s' % node.name, None)
-            if convert_fn and self.should_convert_tag(node.name):
-                text = convert_fn(node, text, convert_as_inline)
+        # apply this tag's final conversion function
+        convert_fn_name = "convert_%s" % re.sub(r"[\[\]:-]", "_", node.name)
+        convert_fn = getattr(self, convert_fn_name, None)
+        if convert_fn and self.should_convert_tag(node.name):
+            text = convert_fn(node, text, convert_as_inline)
 
         return text
 
+    def convert__document_(self, el, text, convert_as_inline):
+        # for BeautifulSoup objects (where node.name == "[document]"), return content results as-is
+        return text
+
     def process_text(self, el):
         text = six.text_type(el) or ''
 
@@ -367,8 +373,8 @@ class MarkdownConverter(object):
         text = all_whitespace_re.sub(' ', text)
         hashes = '#' * n
         if style == ATX_CLOSED:
-            return '\n%s %s %s\n\n' % (hashes, text, hashes)
-        return '\n%s %s\n\n' % (hashes, text)
+            return '\n\n%s %s %s\n\n' % (hashes, text, hashes)
+        return '\n\n%s %s\n\n' % (hashes, text)
 
     def convert_hr(self, el, text, convert_as_inline):
         return '\n\n---\n\n'
@@ -518,13 +524,24 @@ class MarkdownConverter(object):
         cells = el.find_all(['td', 'th'])
         is_headrow = (
             all([cell.name == 'th' for cell in cells])
-            or (not el.previous_sibling and not el.parent.name == 'tbody')
+            or (el.parent.name == 'thead'
+                # avoid multiple tr in thead
+                and len(el.parent.find_all('tr')) == 1)
+        )
+        is_head_row_missing = (
+            (not el.previous_sibling and not el.parent.name == 'tbody')
             or (not el.previous_sibling and el.parent.name == 'tbody' and len(el.parent.parent.find_all(['thead'])) < 1)
         )
         overline = ''
         underline = ''
-        if is_headrow and not el.previous_sibling:
-            # first row and is headline: print headline underline
+        if ((is_headrow
+             or (is_head_row_missing
+                 and self.options['table_infer_header']))
+                and not el.previous_sibling):
+            # first row and:
+            # - is headline or
+            # - headline is missing and header inference is enabled
+            # print headline underline
             full_colspan = 0
             for cell in cells:
                 if 'colspan' in cell.attrs and cell['colspan'].isdigit():
@@ -532,13 +549,16 @@ class MarkdownConverter(object):
                 else:
                     full_colspan += 1
             underline += '| ' + ' | '.join(['---'] * full_colspan) + ' |' + '\n'
-        elif (not el.previous_sibling
-              and (el.parent.name == 'table'
-                   or (el.parent.name == 'tbody'
-                       and not el.parent.previous_sibling))):
+        elif ((is_head_row_missing
+               and not self.options['table_infer_header'])
+              or (not el.previous_sibling
+                  and (el.parent.name == 'table'
+                       or (el.parent.name == 'tbody'
+                           and not el.parent.previous_sibling)))):
+            # headline is missing and header inference is disabled or:
             # first row, not headline, and:
-            # - the parent is table or
-            # - the parent is tbody at the beginning of a table.
+            #  - the parent is table or
+            #  - the parent is tbody at the beginning of a table.
             # print empty headline above this row
             overline += '| ' + ' | '.join([''] * len(cells)) + ' |' + '\n'
             overline += '| ' + ' | '.join(['---'] * len(cells)) + ' |' + '\n'
diff --git a/markdownify/main.py b/markdownify/main.py
index 4e1c874..432efb5 100644
--- a/markdownify/main.py
+++ b/markdownify/main.py
@@ -61,6 +61,10 @@ def main(argv=sys.argv[1:]):
                         "should be converted to markdown images instead, this option can "
                         "be set to a list of parent tags that should be allowed to "
                         "contain inline images.")
+    parser.add_argument('--table-infer-header', dest='table_infer_header',
+                        action='store_true',
+                        help="When a table has no header row (as indicated by '<thead>' "
+                        "or '<th>'), use the first body row as the header row.")
     parser.add_argument('-w', '--wrap', action='store_true',
                         help="Wrap all text paragraphs at --wrap-width characters.")
     parser.add_argument('--wrap-width', type=int, default=80)


Test patch:
diff --git a/tests/test_conversions.py b/tests/test_conversions.py
index cc5ebc7..ba8fcd8 100644
--- a/tests/test_conversions.py
+++ b/tests/test_conversions.py
@@ -127,12 +127,12 @@ def test_em():
 
 
 def test_header_with_space():
-    assert md('<h3>\n\nHello</h3>') == '\n### Hello\n\n'
-    assert md('<h3>Hello\n\n\nWorld</h3>') == '\n### Hello World\n\n'
-    assert md('<h4>\n\nHello</h4>') == '\n#### Hello\n\n'
-    assert md('<h5>\n\nHello</h5>') == '\n##### Hello\n\n'
-    assert md('<h5>\n\nHello\n\n</h5>') == '\n##### Hello\n\n'
-    assert md('<h5>\n\nHello   \n\n</h5>') == '\n##### Hello\n\n'
+    assert md('<h3>\n\nHello</h3>') == '\n\n### Hello\n\n'
+    assert md('<h3>Hello\n\n\nWorld</h3>') == '\n\n### Hello World\n\n'
+    assert md('<h4>\n\nHello</h4>') == '\n\n#### Hello\n\n'
+    assert md('<h5>\n\nHello</h5>') == '\n\n##### Hello\n\n'
+    assert md('<h5>\n\nHello\n\n</h5>') == '\n\n##### Hello\n\n'
+    assert md('<h5>\n\nHello   \n\n</h5>') == '\n\n##### Hello\n\n'
 
 
 def test_h1():
@@ -144,24 +144,24 @@ def test_h2():
 
 
 def test_hn():
-    assert md('<h3>Hello</h3>') == '\n### Hello\n\n'
-    assert md('<h4>Hello</h4>') == '\n#### Hello\n\n'
-    assert md('<h5>Hello</h5>') == '\n##### Hello\n\n'
-    assert md('<h6>Hello</h6>') == '\n###### Hello\n\n'
+    assert md('<h3>Hello</h3>') == '\n\n### Hello\n\n'
+    assert md('<h4>Hello</h4>') == '\n\n#### Hello\n\n'
+    assert md('<h5>Hello</h5>') == '\n\n##### Hello\n\n'
+    assert md('<h6>Hello</h6>') == '\n\n###### Hello\n\n'
     assert md('<h10>Hello</h10>') == md('<h6>Hello</h6>')
     assert md('<hn>Hello</hn>') == md('Hello')
 
 
 def test_hn_chained():
-    assert md('<h1>First</h1>\n<h2>Second</h2>\n<h3>Third</h3>', heading_style=ATX) == '\n# First\n\n## Second\n\n### Third\n\n'
-    assert md('X<h1>First</h1>', heading_style=ATX) == 'X\n# First\n\n'
-    assert md('X<h1>First</h1>', heading_style=ATX_CLOSED) == 'X\n# First #\n\n'
+    assert md('<h1>First</h1>\n<h2>Second</h2>\n<h3>Third</h3>', heading_style=ATX) == '\n\n# First\n\n## Second\n\n### Third\n\n'
+    assert md('X<h1>First</h1>', heading_style=ATX) == 'X\n\n# First\n\n'
+    assert md('X<h1>First</h1>', heading_style=ATX_CLOSED) == 'X\n\n# First #\n\n'
     assert md('X<h1>First</h1>') == 'X\n\nFirst\n=====\n\n'
 
 
 def test_hn_nested_tag_heading_style():
-    assert md('<h1>A <p>P</p> C </h1>', heading_style=ATX_CLOSED) == '\n# A P C #\n\n'
-    assert md('<h1>A <p>P</p> C </h1>', heading_style=ATX) == '\n# A P C\n\n'
+    assert md('<h1>A <p>P</p> C </h1>', heading_style=ATX_CLOSED) == '\n\n# A P C #\n\n'
+    assert md('<h1>A <p>P</p> C </h1>', heading_style=ATX) == '\n\n# A P C\n\n'
 
 
 def test_hn_nested_simple_tag():
@@ -177,9 +177,9 @@ def test_hn_nested_simple_tag():
     ]
 
     for tag, markdown in tag_to_markdown:
-        assert md('<h3>A <' + tag + '>' + tag + '</' + tag + '> B</h3>') == '\n### A ' + markdown + ' B\n\n'
+        assert md('<h3>A <' + tag + '>' + tag + '</' + tag + '> B</h3>') == '\n\n### A ' + markdown + ' B\n\n'
 
-    assert md('<h3>A <br>B</h3>', heading_style=ATX) == '\n### A B\n\n'
+    assert md('<h3>A <br>B</h3>', heading_style=ATX) == '\n\n### A B\n\n'
 
     # Nested lists not supported
     # assert md('<h3>A <ul><li>li1</i><li>l2</li></ul></h3>', heading_style=ATX) == '\n### A li1 li2 B\n\n'
@@ -192,18 +192,23 @@ def test_hn_nested_img():
         ("alt='Alt Text' title='Optional title'", "Alt Text", " \"Optional title\""),
     ]
     for image_attributes, markdown, title in image_attributes_to_markdown:
-        assert md('<h3>A <img src="/path/to/img.jpg" ' + image_attributes + '/> B</h3>') == '\n### A' + (' ' + markdown + ' ' if markdown else ' ') + 'B\n\n'
-        assert md('<h3>A <img src="/path/to/img.jpg" ' + image_attributes + '/> B</h3>', keep_inline_images_in=['h3']) == '\n### A ![' + markdown + '](/path/to/img.jpg' + title + ') B\n\n'
+        assert md('<h3>A <img src="/path/to/img.jpg" ' + image_attributes + '/> B</h3>') == '\n\n### A' + (' ' + markdown + ' ' if markdown else ' ') + 'B\n\n'
+        assert md('<h3>A <img src="/path/to/img.jpg" ' + image_attributes + '/> B</h3>', keep_inline_images_in=['h3']) == '\n\n### A ![' + markdown + '](/path/to/img.jpg' + title + ') B\n\n'
 
 
 def test_hn_atx_headings():
-    assert md('<h1>Hello</h1>', heading_style=ATX) == '\n# Hello\n\n'
-    assert md('<h2>Hello</h2>', heading_style=ATX) == '\n## Hello\n\n'
+    assert md('<h1>Hello</h1>', heading_style=ATX) == '\n\n# Hello\n\n'
+    assert md('<h2>Hello</h2>', heading_style=ATX) == '\n\n## Hello\n\n'
 
 
 def test_hn_atx_closed_headings():
-    assert md('<h1>Hello</h1>', heading_style=ATX_CLOSED) == '\n# Hello #\n\n'
-    assert md('<h2>Hello</h2>', heading_style=ATX_CLOSED) == '\n## Hello ##\n\n'
+    assert md('<h1>Hello</h1>', heading_style=ATX_CLOSED) == '\n\n# Hello #\n\n'
+    assert md('<h2>Hello</h2>', heading_style=ATX_CLOSED) == '\n\n## Hello ##\n\n'
+
+
+def test_hn_newlines():
+    assert md("<h1>H1-1</h1>TEXT<h2>H2-2</h2>TEXT<h1>H1-2</h1>TEXT", heading_style=ATX) == '\n\n# H1-1\n\nTEXT\n\n## H2-2\n\nTEXT\n\n# H1-2\n\nTEXT'
+    assert md('<h1>H1-1</h1>\n<p>TEXT</p>\n<h2>H2-2</h2>\n<p>TEXT</p>\n<h1>H1-2</h1>\n<p>TEXT</p>', heading_style=ATX) == '\n\n# H1-1\n\nTEXT\n\n## H2-2\n\nTEXT\n\n# H1-2\n\nTEXT\n\n'
 
 
 def test_head():
diff --git a/tests/test_custom_converter.py b/tests/test_custom_converter.py
index a3e33ac..adc83f7 100644
--- a/tests/test_custom_converter.py
+++ b/tests/test_custom_converter.py
@@ -2,22 +2,29 @@ from markdownify import MarkdownConverter
 from bs4 import BeautifulSoup
 
 
-class ImageBlockConverter(MarkdownConverter):
+class UnitTestConverter(MarkdownConverter):
     """
-    Create a custom MarkdownConverter that adds two newlines after an image
+    Create a custom MarkdownConverter for unit tests
     """
     def convert_img(self, el, text, convert_as_inline):
+        """Add two newlines after an image"""
         return super().convert_img(el, text, convert_as_inline) + '\n\n'
 
+    def convert_custom_tag(self, el, text, convert_as_inline):
+        """Ensure conversion function is found for tags with special characters in name"""
+        return "FUNCTION USED: %s" % text
 
-def test_img():
+
+def test_custom_conversion_functions():
     # Create shorthand method for conversion
     def md(html, **options):
-        return ImageBlockConverter(**options).convert(html)
+        return UnitTestConverter(**options).convert(html)
 
     assert md('<img src="/path/to/img.jpg" alt="Alt text" title="Optional title" />') == '![Alt text](/path/to/img.jpg "Optional title")\n\n'
     assert md('<img src="/path/to/img.jpg" alt="Alt text" />') == '![Alt text](/path/to/img.jpg)\n\n'
 
+    assert md("<custom-tag>text</custom-tag>") == "FUNCTION USED: text"
+
 
 def test_soup():
     html = '<b>test</b>'
diff --git a/tests/test_tables.py b/tests/test_tables.py
index dcf9ad7..da4bf53 100644
--- a/tests/test_tables.py
+++ b/tests/test_tables.py
@@ -141,6 +141,33 @@ table_head_body_missing_head = """<table>
     </tbody>
 </table>"""
 
+table_head_body_multiple_head = """<table>
+    <thead>
+        <tr>
+            <td>Creator</td>
+            <td>Editor</td>
+            <td>Server</td>
+        </tr>
+        <tr>
+            <td>Operator</td>
+            <td>Manager</td>
+            <td>Engineer</td>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>Bob</td>
+            <td>Oliver</td>
+            <td>Tom</td>
+        </tr>
+        <tr>
+            <td>Thomas</td>
+            <td>Lucas</td>
+            <td>Ethan</td>
+        </tr>
+    </tbody>
+</table>"""
+
 table_missing_text = """<table>
     <thead>
         <tr>
@@ -245,10 +272,28 @@ def test_table():
     assert md(table_with_linebreaks) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith Jackson | 50 |\n| Eve | Jackson Smith | 94 |\n\n'
     assert md(table_with_header_column) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_head_body) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_head_body_multiple_head) == '\n\n|  |  |  |\n| --- | --- | --- |\n| Creator | Editor | Server |\n| Operator | Manager | Engineer |\n| Bob | Oliver | Tom |\n| Thomas | Lucas | Ethan |\n\n'
     assert md(table_head_body_missing_head) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_missing_text) == '\n\n|  | Lastname | Age |\n| --- | --- | --- |\n| Jill |  | 50 |\n| Eve | Jackson | 94 |\n\n'
-    assert md(table_missing_head) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
-    assert md(table_body) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
-    assert md(table_with_caption) == 'TEXT\n\nCaption\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n\n'
+    assert md(table_missing_head) == '\n\n|  |  |  |\n| --- | --- | --- |\n| Firstname | Lastname | Age |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_body) == '\n\n|  |  |  |\n| --- | --- | --- |\n| Firstname | Lastname | Age |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_caption) == 'TEXT\n\nCaption\n\n|  |  |  |\n| --- | --- | --- |\n| Firstname | Lastname | Age |\n\n'
     assert md(table_with_colspan) == '\n\n| Name | | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_with_undefined_colspan) == '\n\n| Name | Age |\n| --- | --- |\n| Jill | Smith |\n\n'
+
+
+def test_table_infer_header():
+    assert md(table, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_html_content, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| **Jill** | *Smith* | [50](#) |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_paragraphs, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_linebreaks, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith Jackson | 50 |\n| Eve | Jackson Smith | 94 |\n\n'
+    assert md(table_with_header_column, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_head_body, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_head_body_multiple_head, table_infer_header=True) == '\n\n| Creator | Editor | Server |\n| --- | --- | --- |\n| Operator | Manager | Engineer |\n| Bob | Oliver | Tom |\n| Thomas | Lucas | Ethan |\n\n'
+    assert md(table_head_body_missing_head, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_missing_text, table_infer_header=True) == '\n\n|  | Lastname | Age |\n| --- | --- | --- |\n| Jill |  | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_missing_head, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_body, table_infer_header=True) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_caption, table_infer_header=True) == 'TEXT\n\nCaption\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n\n'
+    assert md(table_with_colspan, table_infer_header=True) == '\n\n| Name | | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_undefined_colspan, table_infer_header=True) == '\n\n| Name | Age |\n| --- | --- |\n| Jill | Smith |\n\n'