Repository: jugglerchris/rust-html2text
Pull request #228 (2025-06-29 11:09:49)

Bugfix 20250629
Fix whitespace and wrapping issues: #224 and #226 .


Patch:
diff --git a/src/css.rs b/src/css.rs
index 49384eb..2c5add8 100644
--- a/src/css.rs
+++ b/src/css.rs
@@ -721,7 +721,7 @@ impl std::fmt::Display for StyleData {
 pub(crate) mod dom_extract {
     use std::io::Write;
 
-    use crate::{expanded_name, local_name, namespace_url, ns};
+    use crate::{expanded_name, local_name, ns};
 
     use crate::{
         markup5ever_rcdom::{
diff --git a/src/lib.rs b/src/lib.rs
index 5dcad78..288cd45 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -60,6 +60,48 @@ mod macros;
 pub mod css;
 pub mod render;
 
+/// Extra methods on chars for dealing with special cases with wrapping and whitespace.
+trait WhitespaceExt {
+    /// Returns whether this character always takes space. This is true for non-whitespace and
+    /// non-breaking spaces.
+    fn always_takes_space(&self) -> bool;
+
+    /// Returns true if a word before this character is allowed. This includes most whitespace
+    /// (but not non-breaking space).
+    fn is_wordbreak_point(&self) -> bool;
+}
+
+impl WhitespaceExt for char {
+    fn always_takes_space(&self) -> bool {
+        match *self {
+            '\u{A0}' => true,
+            c if !c.is_whitespace() => true,
+            _ => false,
+        }
+    }
+
+    fn is_wordbreak_point(&self) -> bool {
+        match *self {
+            '\u{00A0}' => false,
+            '\u{200b}' => true,
+            c if c.is_whitespace() => true,
+            _ => false,
+        }
+    }
+}
+
+/// Extra methods for strings
+trait StrExt {
+    /// Trims leading/trailing whitespace expect for hard spaces.
+    fn trim_collapsible_ws(&self) -> &str;
+}
+
+impl StrExt for str {
+    fn trim_collapsible_ws(&self) -> &str {
+        self.trim_matches(|c: char| !c.always_takes_space())
+    }
+}
+
 #[cfg(feature = "css_ext")]
 /// Text style information.
 #[derive(Clone, Debug)]
@@ -604,7 +646,7 @@ impl RenderTable {
                 colno += cell.colspan;
             }
         }
-        let size = sizes.iter().map(|s| s.size).sum(); // Include borders?
+        let size = sizes.iter().map(|s| s.size).sum::<usize>() + self.num_columns.saturating_sub(1);
         let min_width = sizes.iter().map(|s| s.min_width).sum::<usize>() + self.num_columns - 1;
         let result = SizeEstimate {
             size,
@@ -727,20 +769,22 @@ impl RenderNode {
                 use unicode_width::UnicodeWidthChar;
                 let mut len = 0;
                 let mut in_whitespace = false;
-                for c in t.trim().chars() {
-                    let is_ws = c.is_whitespace();
-                    if !is_ws {
+                for c in t.trim_collapsible_ws().chars() {
+                    let is_collapsible_ws = !c.always_takes_space();
+                    if !is_collapsible_ws {
                         len += UnicodeWidthChar::width(c).unwrap_or(0);
                         // Count the preceding whitespace as one.
                         if in_whitespace {
                             len += 1;
                         }
                     }
-                    in_whitespace = is_ws;
+                    in_whitespace = is_collapsible_ws;
                 }
-                // Add one for preceding whitespace.
-                if let Some(true) = t.chars().next().map(|c| c.is_whitespace()) {
-                    len += 1;
+                // Add one for preceding whitespace, unless the node is otherwise empty.
+                if let Some(true) = t.chars().next().map(|c| !c.always_takes_space()) {
+                    if len > 0 {
+                        len += 1;
+                    }
                 }
                 if let Img(_, _) = self.info {
                     len += 2;
@@ -882,27 +926,31 @@ impl RenderNode {
         indent: usize,
         style: &ComputedStyle,
     ) -> std::result::Result<(), std::fmt::Error> {
-        write!(f, "{:indent$}[Style:", "")?;
+        use std::fmt::Write;
+        let mut stylestr = String::new();
 
         #[cfg(feature = "css")]
         {
             if let Some(col) = style.colour.val() {
-                write!(f, " colour={:?}", col)?;
+                write!(&mut stylestr, " colour={:?}", col)?;
             }
             if let Some(col) = style.bg_colour.val() {
-                write!(f, " bg_colour={:?}", col)?;
+                write!(&mut stylestr, " bg_colour={:?}", col)?;
             }
             if let Some(val) = style.display.val() {
-                write!(f, " disp={:?}", val)?;
+                write!(&mut stylestr, " disp={:?}", val)?;
             }
         }
         if let Some(ws) = style.white_space.val() {
-            write!(f, " white_space={:?}", ws)?;
+            write!(&mut stylestr, " white_space={:?}", ws)?;
         }
         if style.internal_pre {
-            write!(f, " internal_pre")?;
+            write!(&mut stylestr, " internal_pre")?;
+        }
+        if !stylestr.is_empty() {
+            writeln!(f, "{:indent$}[Style:{stylestr}", "")?;
         }
-        writeln!(f)
+        Ok(())
     }
     fn write_self(
         &self,
@@ -1921,6 +1969,9 @@ fn process_dom_node<T: Write>(
                     Some(RenderNode::new_styled(Block(cs), computed))
                 }),
                 expanded_name!(html "br") => Finished(RenderNode::new_styled(Break, computed)),
+                expanded_name!(html "wbr") => {
+                    Finished(RenderNode::new_styled(Text("\u{200b}".into()), computed))
+                }
                 expanded_name!(html "table") => table_to_render_tree(input, computed, err_out),
                 expanded_name!(html "thead") | expanded_name!(html "tbody") => {
                     tbody_to_render_tree(input, computed, err_out)
@@ -2524,7 +2575,6 @@ fn render_table_tree<T: Write, D: TextDecorator>(
             colno += cell.colspan;
         }
     }
-    // TODO: remove empty columns
     let tot_size: usize = col_sizes.iter().map(|est| est.size).sum();
     let min_size: usize = col_sizes.iter().map(|est| est.min_width).sum::<usize>()
         + col_sizes.len().saturating_sub(1);
diff --git a/src/render/text_renderer.rs b/src/render/text_renderer.rs
index a6caf70..4c0c1ae 100644
--- a/src/render/text_renderer.rs
+++ b/src/render/text_renderer.rs
@@ -5,6 +5,7 @@
 
 use crate::Colour;
 use crate::WhiteSpace;
+use crate::WhitespaceExt as _;
 
 use super::Renderer;
 use super::Result;
@@ -326,6 +327,62 @@ impl<T: Debug + Eq + PartialEq + Clone + Default> TaggedLine<T> {
             self.push_ws(width - my_width, tag);
         }
     }
+
+    /// Remove leading whitespace.
+    fn remove_leading_whitespace(&mut self) {
+        let mut pieces_to_remove = 0;
+        let mut width_removed = 0;
+        for element in &mut self.v {
+            if let TaggedLineElement::Str(piece) = element {
+                let trimmed = piece.s.trim_start();
+                let tlen = trimmed.len();
+                let toffset = piece.s.len() - tlen;
+                if toffset == 0 {
+                    // No more leading whitespace
+                    break;
+                }
+                if tlen == 0 {
+                    // The piece would be empty.
+                    pieces_to_remove += 1;
+                    width_removed += piece.width();
+                } else {
+                    // Shrink it to fit.
+                    let orig_width = piece.width();
+                    piece.s.replace_range(0..toffset, "");
+                    let new_width = piece.width();
+                    width_removed += orig_width - new_width;
+                    break;
+                }
+            } else {
+                // Don't do anything to fragments
+                break;
+            }
+        }
+        if pieces_to_remove > 0 {
+            self.v = self.v.split_off(pieces_to_remove);
+        }
+        self.len -= width_removed;
+    }
+
+    /// Remove tralling spaces
+    fn remove_trailing_spaces(&mut self) {
+        while let Some(TaggedLineElement::Str(piece)) = self.v.last_mut() {
+            let trimmed = piece.s.trim_end_matches(' ');
+            let tlen = trimmed.len();
+            if tlen == 0 {
+                // Remove the whole last element.
+                self.len -= piece.width();
+                self.v.pop();
+            } else if tlen == piece.s.len() {
+                // We're done.
+                break;
+            } else {
+                self.len -= piece.width() - trimmed.width();
+                piece.s.replace_range(tlen.., "");
+                break;
+            }
+        }
+    }
 }
 
 impl<T> IntoIterator for TaggedLine<T> {
@@ -404,22 +461,34 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
                 html_trace!("linelen increased by wordlen to {}", self.line.len);
             } else {
                 html_trace!("Not enough space");
-                // The column position inside (whitespace + word)
-                if !ws_mode.do_wrap() {
-                    // We're not word-wrapping, so output any portion that still
-                    // fits.
-                    if self.wslen >= space_in_line {
-                        // Skip the whitespace
-                        self.wslen -= space_in_line;
-                    } else if self.wslen > 0 {
-                        self.line
-                            .push_ws(self.wslen, &self.spacetag.take().unwrap());
+                match ws_mode {
+                    WhiteSpace::Pre => {
+                        // We're not word-wrapping, so output any portion that still
+                        // fits.
+                        if self.wslen >= space_in_line {
+                            // Skip the whitespace
+                            self.wslen -= space_in_line;
+                        } else if self.wslen > 0 {
+                            self.line
+                                .push_ws(self.wslen, &self.spacetag.take().unwrap());
+                            self.wslen = 0;
+                        }
+                    }
+                    WhiteSpace::Normal => {
+                        // We're word-wrapping, so discard any whitespace.
+                        self.spacetag = None;
                         self.wslen = 0;
                     }
-                } else {
-                    // We're word-wrapping, so discard any whitespace.
-                    self.spacetag = None;
-                    self.wslen = 0;
+                    WhiteSpace::PreWrap => {
+                        // Preserving whitespace except at wrap points.
+                        self.spacetag = None;
+                        self.wslen = 0;
+
+                        // Remove whitespace at the start of the word.
+                        self.word.remove_leading_whitespace();
+                        // And remove spaces at the end of the line.
+                        self.line.remove_trailing_spaces();
+                    }
                 }
                 /* Start a new line */
                 self.flush_line();
@@ -580,6 +649,8 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
         // 2a. If the word gets too long for the line
         // 2b. If we get to more whitespace, output the first whitespace and the word
         //     and continue.
+        //
+        // In "pre" mode, we treat whitespace other than newlines as word characters.
         let mut tag = if self.pre_wrapped { wrap_tag } else { main_tag };
         for c in text.chars() {
             html_trace!(
@@ -590,17 +661,18 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
                 self.wslen,
                 self.line
             );
-            if c.is_whitespace() && self.wordlen > 0 {
+            if c.is_wordbreak_point() && self.wordlen > 0 && ws_mode != WhiteSpace::Pre {
                 self.flush_word(ws_mode)?;
             }
 
-            if c.is_whitespace() {
-                // We're just building up whitespace.
+            if c == '\u{200b}' {
+                // This is a zero-width space.  As we're doing the formatting, we can just omit it.
+                continue;
+            } else if !c.always_takes_space() {
                 if ws_mode.preserve_whitespace() {
                     match c {
                         '\n' => {
-                            // End of line.  We have no words here, so just finish
-                            // the line.
+                            self.flush_word(ws_mode)?;
                             self.force_flush_line();
                             self.wslen = 0;
                             self.spacetag = None;
@@ -609,8 +681,11 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
                             tag = main_tag;
                         }
                         '\t' => {
+                            // Flush the word.
+                            self.flush_word(ws_mode)?;
+
                             let tab_stop = 8;
-                            let mut pos = self.line.len + self.wslen;
+                            let mut pos = self.line.len + self.wordlen + self.wslen;
                             let mut at_least_one_space = false;
                             while pos % tab_stop != 0 || !at_least_one_space {
                                 if pos >= self.width {
@@ -625,25 +700,10 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
                         }
                         _ => {
                             if let Some(cwidth) = UnicodeWidthChar::width(c) {
-                                if (self.line.len + self.wslen + cwidth) > self.width {
-                                    // In any case we can discard whitespace we've
-                                    // built up, as it will be at the end of the line.
-                                    self.wslen = 0;
-
-                                    self.flush_line();
-                                    if ws_mode.do_wrap() {
-                                        // We're handling wrapping, so collapse
-                                        self.pre_wrapped = false;
-                                    } else {
-                                        // Manual wrapping, keep the space.
-                                        self.wslen += cwidth;
-                                        self.spacetag = Some(tag.clone());
-                                        self.pre_wrapped = true;
-                                    }
-                                } else {
-                                    self.spacetag = Some(tag.clone());
-                                    self.wslen += cwidth;
-                                }
+                                // Add the character, unless we're in prewrap mode and character
+                                // is leading whitespace.
+                                self.word.push_char(c, tag);
+                                self.wordlen += cwidth;
                             }
                         }
                     }


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
index 68848e5..74a3b1d 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -910,6 +910,54 @@ full screen width
     );
 }
 
+#[test]
+fn test_wrap_nbsp_unicode() {
+    // Use nbsp unicode character
+    test_html(
+        b"<p>Don't break a line between the words Foo\xc2\xa0and\xc2\xa0Bar</p>",
+        "Don't break a line between the words
+Foo\u{a0}and\u{a0}Bar
+",
+        40,
+    );
+}
+
+#[test]
+fn test_wrap_nbsp_ent() {
+    // Use `&nbsp;` HTML entity
+    test_html(
+        br#"<p>Don't break a line between the words Foo&nbsp;and&nbsp;Bar</p>"#,
+        "Don't break a line between the words
+Foo\u{a0}and\u{a0}Bar
+",
+        40,
+    );
+}
+
+#[test]
+fn test_dowrap_unicode() {
+    // Use Unicode nbsp
+    test_html(
+        b"<p>Do break a line somewhere in foo\xe2\x80\x8bbar\xe2\x80\x8bfoo\xe2\x80\x8bbar\xe2\x80\x8bfoo\xe2\x80\x8bbar\xe2\x80\x8bfoo\xe2\x80\x8bbar\xe2\x80\x8bfoo\xe2\x80\x8bbar\xe2\x80\x8bfoo\xe2\x80\x8bbar</p>",
+        r#"Do break a line somewhere in foobarfoo
+barfoobarfoobarfoobarfoobar
+"#,
+        40,
+    );
+}
+
+#[test]
+fn test_dowrap_wbr() {
+    // Use `<wbr>` HTML element
+    test_html(
+        b"<p>Do break a line somewhere in foo<wbr>bar<wbr>foo<wbr>bar<wbr>foo<wbr>bar<wbr>foo<wbr>bar<wbr>foo<wbr>bar<wbr>foo<wbr>bar</p>",
+        r#"Do break a line somewhere in foobarfoo
+barfoobarfoobarfoobarfoobar
+"#,
+        40,
+    );
+}
+
 #[test]
 fn test_nested_ul() {
     test_html(
@@ -1407,7 +1455,6 @@ fn test_em_strong() {
 }
 
 #[test]
-#[ignore] // Not yet fixed!
 fn test_nbsp_indent() {
     test_html(
         br##"
@@ -1415,10 +1462,10 @@ fn test_nbsp_indent() {
    <div>&nbsp;Indented</div>
    <div>&nbsp;&nbsp;Indented again</div>
 "##,
-        r#"Top
-Indented
-Indented again
-"#,
+        "Top
+\u{a0}Indented
+\u{a0}\u{a0}Indented again
+",
         21,
     );
 }
@@ -1456,13 +1503,9 @@ hi
     .take(rpt - 3)
     .collect::<Vec<_>>()
     .concat()
-        + r#"──┬────
-hi│hi  
-  │////
-  │──  
-  │hi  
-  │──  
-──┴────
+        + r#"──┬──┬───
+hi│hi│hi 
+──┴──┴───
 "# + &"──────────\n".repeat(rpt - 3);
     test_html(html.as_bytes(), &result, 10);
 }
@@ -1621,7 +1664,7 @@ Bar
 
 #[test]
 fn test_pre_emptyline() {
-    test_html(br#"<pre>X<span id="i"> </span></pre>"#, "X\n", 10);
+    test_html(br#"<pre>X<span id="i"> </span></pre>"#, "X \n", 10);
 }
 
 #[test]
@@ -1951,6 +1994,25 @@ fn test_table_empty_single_row_empty_cell() {
     );
 }
 
+#[test]
+fn test_table_empty_single_row_ws_cell() {
+    test_html(
+        br##"
+   <table><tr><td> </td></tr></table>
+"##,
+        r#""#,
+        12,
+    );
+    test_html(
+        br##"
+   <table><tr><td>
+</td></tr></table>
+"##,
+        r#""#,
+        12,
+    );
+}
+
 #[test]
 fn test_renderer_zero_width() {
     test_html_err(
@@ -2002,12 +2064,12 @@ fn test_issue_54_oob() {
     </table>
 </body>
 "##,
-        r#"─┬──────┬─
- │Blah  │ 
+        "─┬──────┬─
+\u{a0}│Blah  │\u{a0}
  │blah  │ 
  │blah  │ 
 ─┴──────┴─
-"#,
+",
         10,
     );
 }
@@ -3145,6 +3207,17 @@ at  line  breaks
             80,
         );
     }
+
+    #[test]
+    fn test_wrap_nbsp_style() {
+        test_html_css(
+            br#"<p>Don't break a line between the words <span style="white-space: pre">Foo and Bar</span></p>"#,
+            r#"Don't break a line between the words
+Foo and Bar
+"#,
+40,
+        );
+    }
 }
 
 #[cfg(feature = "css_ext")]