Repository: matthewwithanm/python-markdownify
Pull request #83 (2023-01-08 08:28:48)

Support conversion of header rows in tables without th tag
Hello !
As I stated in the title, I would like to see support for conversion of header rows in tables without th tags.
The original test code was implemented as not supporting the conversion of header rows in tables with missing th tags, so I would also like to know what the intent was.
If there are any problems with this fix, please let me know.

huuyafwww: This is what I have modified.

### sample1

missing th, without tbody and thead.

```html
<table>
  <tr>
    <td>key</td>
    <td>name</td>
  </tr>
  <tr>
    <td>hoge</td>
    <td>foo</td>
  </tr>
</table>
```

#### before

```
|  |  |
| --- | --- |
| key | name |
| hoge | foo |
```

#### after

```
| key | name |
| --- | --- |
| hoge | foo |
```

### sample2

missing th, with tbody, without thead.

```html
<table>
  <tbody>
    <tr>
      <td>key</td>
      <td>name</td>
    </tr>
    <tr>
      <td>hoge</td>
      <td>foo</td>
    </tr>
  </tbody>
</table>
```

#### before

```
|  |  |
| --- | --- |
| key | name |
| hoge | foo |
```

#### after

```
| key | name |
| --- | --- |
| hoge | foo |
```

### sample3

missing th, with tbody and thead.

```html
<table>
  <thead>
    <tr>
      <td>key</td>
      <td>name</td>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>hoge</td>
      <td>foo</td>
    </tr>
  </tbody>
</table>
```

#### before

```
| key | name |
| hoge | foo |
```

#### after

```
| key | name |
| --- | --- |
| hoge | foo |
```
SurajDonthi: @matthewwithanm This is a much-needed bug fix for Markdown tables. Please merge this PR asap.
huuyafwww: @SurajDonthi 

Thanks for your support!
To begin with, though, this modification is required as a markdown table,
I needed this fix right away, so I used the following code instead, based on "[Creating a Custom Converter](https://github.com/matthewwithanm/python-markdownify#creating-custom-converters)" in README.md.

```python
from markdownify import MarkdownConverter

class CustomMarkdownConverter(MarkdownConverter):

  def convert_tr(self, el, text, convert_as_inline):
    cells = el.find_all(['td', 'th'])
    is_headrow = all([cell.name == 'th' for cell in cells])
    is_headrow = (
      all([cell.name == 'th' for cell in cells])
      or (not el.previous_sibling and not el.parent.name == 'tbody')
      or (not el.previous_sibling and el.parent.name == 'tbody' and len(el.parent.parent.find_all(['thead'])) < 1)
    )
    overline = ''
    underline = ''
    if is_headrow and not el.previous_sibling:
      # first row and is headline: print headline underline
      underline += '| ' + ' | '.join(['---'] * len(cells)) + ' |' + '\n'
    elif (not el.previous_sibling
      and (el.parent.name == 'table'
          or (el.parent.name == 'tbody'
            and not el.parent.previous_sibling))):
      # first row, not headline, and:
      # - the parent is table or
      # - the parent is tbody at the beginning of a table.
      # print empty headline above this row
      overline += '| ' + ' | '.join([''] * len(cells)) + ' |' + '\n'
      overline += '| ' + ' | '.join(['---'] * len(cells)) + ' |' + '\n'
    return overline + '|' + text + '\n' + underline


def convert(html, **options):
  return CustomMarkdownConverter(**options).convert(html)


html = '''
<table>
  <tr>
    <td>key</td>
    <td>name</td>
  </tr>
  <tr>
    <td>hoge</td>
    <td>foo</td>
  </tr>
</table>
'''

convert(
  html,
  # Options for markdownify,
)
```


Patch:
diff --git a/README.rst b/README.rst
index 06f319b..51888ea 100644
--- a/README.rst
+++ b/README.rst
@@ -156,7 +156,12 @@ Creating Custom Converters
 
 If you have a special usecase that calls for a special conversion, you can
 always inherit from ``MarkdownConverter`` and override the method you want to
-change:
+change.
+The function that handles a HTML tag named ``abc`` is called
+``convert_abc(self, el, text, convert_as_inline)`` and returns a string
+containing the converted HTML tag.
+The ``MarkdownConverter`` object will handle the conversion based on the
+function names:
 
 .. code:: python
 
@@ -173,6 +178,21 @@ change:
     def md(html, **options):
         return ImageBlockConverter(**options).convert(html)
 
+.. code:: python
+
+    from markdownify import MarkdownConverter
+
+    class IgnoreParagraphsConverter(MarkdownConverter):
+        """
+        Create a custom MarkdownConverter that ignores paragraphs
+        """
+        def convert_p(self, el, text, convert_as_inline):
+            return ''
+
+    # Create shorthand method for conversion
+    def md(html, **options):
+        return IgnoreParagraphsConverter(**options).convert(html)
+
 
 Command Line Interface
 ======================
diff --git a/markdownify/__init__.py b/markdownify/__init__.py
index e15ecd4..b67c32b 100644
--- a/markdownify/__init__.py
+++ b/markdownify/__init__.py
@@ -152,13 +152,12 @@ class MarkdownConverter(object):
     def process_text(self, el):
         text = six.text_type(el) or ''
 
-        # dont remove any whitespace when handling pre or code in pre
-        if not (el.parent.name == 'pre'
-                or (el.parent.name == 'code'
-                    and el.parent.parent.name == 'pre')):
+        # normalize whitespace if we're not inside a preformatted element
+        if not el.find_parent('pre'):
             text = whitespace_re.sub(' ', text)
 
-        if el.parent.name != 'code' and el.parent.name != 'pre':
+        # escape special characters if we're not inside a preformatted or code element
+        if not el.find_parent(['pre', 'code', 'kbd', 'samp']):
             text = self.escape(text)
 
         # remove trailing whitespaces if any of the following condition is true:
@@ -238,7 +237,7 @@ class MarkdownConverter(object):
         if convert_as_inline:
             return text
 
-        return '\n' + (line_beginning_re.sub('> ', text) + '\n\n') if text else ''
+        return '\n' + (line_beginning_re.sub('> ', text.strip()) + '\n\n') if text else ''
 
     def convert_br(self, el, text, convert_as_inline):
         if convert_as_inline:
@@ -266,7 +265,7 @@ class MarkdownConverter(object):
             return text
 
         style = self.options['heading_style'].lower()
-        text = text.rstrip()
+        text = text.strip()
         if style == UNDERLINED and n <= 2:
             line = '=' if n == 1 else '-'
             return self.underline(text, line)
@@ -351,6 +350,12 @@ class MarkdownConverter(object):
 
         return '\n```%s\n%s\n```\n' % (code_language, text)
 
+    def convert_script(self, el, text, convert_as_inline):
+        return ''
+
+    def convert_style(self, el, text, convert_as_inline):
+        return ''
+
     convert_s = convert_del
 
     convert_strong = convert_b
@@ -365,14 +370,18 @@ class MarkdownConverter(object):
         return '\n\n' + text + '\n'
 
     def convert_td(self, el, text, convert_as_inline):
-        return ' ' + text + ' |'
+        return ' ' + text.strip().replace("\n", " ") + ' |'
 
     def convert_th(self, el, text, convert_as_inline):
         return ' ' + text + ' |'
 
     def convert_tr(self, el, text, convert_as_inline):
         cells = el.find_all(['td', 'th'])
-        is_headrow = all([cell.name == 'th' for cell in cells])
+        is_headrow = (
+            all([cell.name == 'th' for cell in cells])
+            or (not el.previous_sibling and not el.parent.name == 'tbody')
+            or (not el.previous_sibling and el.parent.name == 'tbody' and len(el.parent.parent.find_all(['thead'])) < 1)
+        )
         overline = ''
         underline = ''
         if is_headrow and not el.previous_sibling:


Test patch:
diff --git a/tests/test_conversions.py b/tests/test_conversions.py
index da78649..ae56837 100644
--- a/tests/test_conversions.py
+++ b/tests/test_conversions.py
@@ -52,6 +52,12 @@ def test_b_spaces():
 
 def test_blockquote():
     assert md('<blockquote>Hello</blockquote>') == '\n> Hello\n\n'
+    assert md('<blockquote>\nHello\n</blockquote>') == '\n> Hello\n\n'
+
+
+def test_blockquote_with_nested_paragraph():
+    assert md('<blockquote><p>Hello</p></blockquote>') == '\n> Hello\n\n'
+    assert md('<blockquote><p>Hello</p><p>Hello again</p></blockquote>') == '\n> Hello\n> \n> Hello again\n\n'
 
 
 def test_blockquote_with_paragraph():
@@ -60,7 +66,7 @@ def test_blockquote_with_paragraph():
 
 def test_blockquote_nested():
     text = md('<blockquote>And she was like <blockquote>Hello</blockquote></blockquote>')
-    assert text == '\n> And she was like \n> > Hello\n> \n> \n\n'
+    assert text == '\n> And she was like \n> > Hello\n\n'
 
 
 def test_br():
@@ -70,7 +76,12 @@ def test_br():
 
 def test_code():
     inline_tests('code', '`')
-    assert md('<code>this_should_not_escape</code>') == '`this_should_not_escape`'
+    assert md('<code>*this_should_not_escape*</code>') == '`*this_should_not_escape*`'
+    assert md('<kbd>*this_should_not_escape*</kbd>') == '`*this_should_not_escape*`'
+    assert md('<samp>*this_should_not_escape*</samp>') == '`*this_should_not_escape*`'
+    assert md('<code><span>*this_should_not_escape*</span></code>') == '`*this_should_not_escape*`'
+    assert md('<code>this  should\t\tnormalize</code>') == '`this should normalize`'
+    assert md('<code><span>this  should\t\tnormalize</span></code>') == '`this should normalize`'
 
 
 def test_del():
@@ -85,6 +96,14 @@ def test_em():
     inline_tests('em', '*')
 
 
+def test_header_with_space():
+    assert md('<h3>\n\nHello</h3>') == '### Hello\n\n'
+    assert md('<h4>\n\nHello</h4>') == '#### Hello\n\n'
+    assert md('<h5>\n\nHello</h5>') == '##### Hello\n\n'
+    assert md('<h5>\n\nHello\n\n</h5>') == '##### Hello\n\n'
+    assert md('<h5>\n\nHello   \n\n</h5>') == '##### Hello\n\n'
+
+
 def test_h1():
     assert md('<h1>Hello</h1>') == 'Hello\n=====\n\n'
 
@@ -187,7 +206,18 @@ def test_p():
 def test_pre():
     assert md('<pre>test\n    foo\nbar</pre>') == '\n```\ntest\n    foo\nbar\n```\n'
     assert md('<pre><code>test\n    foo\nbar</code></pre>') == '\n```\ntest\n    foo\nbar\n```\n'
-    assert md('<pre>this_should_not_escape</pre>') == '\n```\nthis_should_not_escape\n```\n'
+    assert md('<pre>*this_should_not_escape*</pre>') == '\n```\n*this_should_not_escape*\n```\n'
+    assert md('<pre><span>*this_should_not_escape*</span></pre>') == '\n```\n*this_should_not_escape*\n```\n'
+    assert md('<pre>\t\tthis  should\t\tnot  normalize</pre>') == '\n```\n\t\tthis  should\t\tnot  normalize\n```\n'
+    assert md('<pre><span>\t\tthis  should\t\tnot  normalize</span></pre>') == '\n```\n\t\tthis  should\t\tnot  normalize\n```\n'
+
+
+def test_script():
+    assert md('foo <script>var foo=42;</script> bar') == 'foo  bar'
+
+
+def test_style():
+    assert md('foo <style>h1 { font-size: larger }</style> bar') == 'foo  bar'
 
 
 def test_s():
diff --git a/tests/test_tables.py b/tests/test_tables.py
index e0c07ea..ebbb146 100644
--- a/tests/test_tables.py
+++ b/tests/test_tables.py
@@ -57,6 +57,26 @@ table_with_paragraphs = """<table>
     </tr>
 </table>"""
 
+table_with_linebreaks = """<table>
+    <tr>
+        <th>Firstname</th>
+        <th>Lastname</th>
+        <th>Age</th>
+    </tr>
+    <tr>
+        <td>Jill</td>
+        <td>Smith
+        Jackson</td>
+        <td>50</td>
+    </tr>
+    <tr>
+        <td>Eve</td>
+        <td>Jackson
+        Smith</td>
+        <td>94</td>
+    </tr>
+</table>"""
+
 
 table_with_header_column = """<table>
     <tr>
@@ -99,6 +119,28 @@ table_head_body = """<table>
     </tbody>
 </table>"""
 
+table_head_body_missing_head = """<table>
+    <thead>
+        <tr>
+            <td>Firstname</td>
+            <td>Lastname</td>
+            <td>Age</td>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>Jill</td>
+            <td>Smith</td>
+            <td>50</td>
+        </tr>
+        <tr>
+            <td>Eve</td>
+            <td>Jackson</td>
+            <td>94</td>
+        </tr>
+    </tbody>
+</table>"""
+
 table_missing_text = """<table>
     <thead>
         <tr>
@@ -164,8 +206,10 @@ def test_table():
     assert md(table) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_with_html_content) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| **Jill** | *Smith* | [50](#) |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_with_paragraphs) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_with_linebreaks) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith  Jackson | 50 |\n| Eve | Jackson  Smith | 94 |\n\n'
     assert md(table_with_header_column) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_head_body) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_head_body_missing_head) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
     assert md(table_missing_text) == '\n\n|  | Lastname | Age |\n| --- | --- | --- |\n| Jill |  | 50 |\n| Eve | Jackson | 94 |\n\n'
-    assert md(table_missing_head) == '\n\n|  |  |  |\n| --- | --- | --- |\n| Firstname | Lastname | Age |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
-    assert md(table_body) == '\n\n|  |  |  |\n| --- | --- | --- |\n| Firstname | Lastname | Age |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_missing_head) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'
+    assert md(table_body) == '\n\n| Firstname | Lastname | Age |\n| --- | --- | --- |\n| Jill | Smith | 50 |\n| Eve | Jackson | 94 |\n\n'