Repository: jugglerchris/rust-html2text
Pull request #204 (2025-03-02 21:58:48)

Some bugfixes
Fix bug #202: fragments lost
Fix bug #203: infinite loop with some tab handling.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f1844a8..3cbbce2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,12 @@ Possible log types:
 
 ### Latest
 
+### 0.14.1
+
+- [fixed] An issue with `FragmentStart`s being lost (thanks toiletbril)
+- [fixed] An infinite loop if tabs inside `<pre>` wrapped past the width
+  (thanks nshp)
+
 ### 0.14.0
 
 - [changed] Various small refactors (thanks sftse)
diff --git a/Cargo.toml b/Cargo.toml
index 016f960..852fddb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "html2text"
-version = "0.14.0"
+version = "0.14.1"
 authors = ["Chris Emerson <github@mail.nosreme.org>"]
 description = "Render HTML as plain text."
 repository = "https://github.com/jugglerchris/rust-html2text/"
diff --git a/src/render/text_renderer.rs b/src/render/text_renderer.rs
index 21d139c..fa6775d 100644
--- a/src/render/text_renderer.rs
+++ b/src/render/text_renderer.rs
@@ -233,6 +233,14 @@ impl<T: Debug + Eq + PartialEq + Clone + Default> TaggedLine<T> {
         use self::TaggedLineElement::Str;
 
         self.len += UnicodeWidthStr::width(ts.s.as_str());
+
+        if let Some(Str(ts1)) = self.v.get_mut(0) {
+            if ts1.tag == ts.tag {
+                // Combine into one TaggedString
+                ts1.s.insert_str(0, &ts.s);
+                return;
+            }
+        }
         self.v.insert(0, Str(ts));
     }
 
@@ -320,6 +328,15 @@ impl<T: Debug + Eq + PartialEq + Clone + Default> TaggedLine<T> {
     }
 }
 
+impl<T> IntoIterator for TaggedLine<T> {
+    type Item = TaggedLineElement<T>;
+    type IntoIter = <Vec<TaggedLineElement<T>> as IntoIterator>::IntoIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.v.into_iter()
+    }
+}
+
 /// A type to build up wrapped text, allowing extra metadata for
 /// spans.
 #[derive(Debug, Clone)]
@@ -531,6 +548,15 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
     }
     */
 
+    /// If there are any pending fragment starts, return them.
+    pub fn take_trailing_fragments(&mut self) -> Vec<TaggedLineElement<T>> {
+        if self.word.is_empty() {
+            std::mem::take(&mut self.word).v
+        } else {
+            Default::default()
+        }
+    }
+
     /// Consume self and return vector of lines including annotations.
     pub fn into_lines(mut self) -> Result<Vec<TaggedLine<T>>> {
         self.flush()?;
@@ -589,6 +615,7 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
                             while pos % tab_stop != 0 || !at_least_one_space {
                                 if pos >= self.width {
                                     self.flush_line();
+                                    pos = 0;
                                 } else {
                                     self.line.push_char(' ', tag);
                                     pos += 1;
@@ -964,7 +991,10 @@ pub(crate) struct SubRenderer<D: TextDecorator> {
     width: usize,
     /// Rendering options
     pub options: RenderOptions,
+    /// The currently generated lines
     lines: LinkedList<RenderLine<Vec<D::Annotation>>>,
+    /// FragmentStart items which have not yet been output.
+    pending_frags: Vec<TaggedLineElement<Vec<D::Annotation>>>,
     /// True at the end of a block, meaning we should add
     /// a blank line if any other text is added.
     at_block_end: bool,
@@ -1106,14 +1136,47 @@ impl<D: TextDecorator> SubRenderer<D> {
             ws_stack: Vec::new(),
             pre_depth: 0,
             text_filter_stack: Vec::new(),
+            pending_frags: Default::default(),
+        }
+    }
+
+    /// Append a line to the output.
+    /// Any pending fragments will be prepended to a non-border line.
+    fn add_line(&mut self, line: RenderLine<Vec<D::Annotation>>) {
+        if !self.pending_frags.is_empty() {
+            match line {
+                RenderLine::Text(tagged_line) => {
+                    let mut tl = TaggedLine::new();
+                    for frag in std::mem::take(&mut self.pending_frags) {
+                        tl.push(frag);
+                    }
+                    for part in tagged_line.into_iter() {
+                        tl.push(part);
+                    }
+                    self.lines.push_back(RenderLine::Text(tl));
+                    return;
+                }
+                RenderLine::Line(..) => (),
+            }
+        }
+        self.lines.push_back(line);
+    }
+
+    /// Append multiple lines to the output
+    /// Any pending fragments will be prepended to a non-border line.
+    fn extend_lines(&mut self, lines: impl IntoIterator<Item = RenderLine<Vec<D::Annotation>>>) {
+        for line in lines {
+            self.add_line(line);
         }
     }
 
     /// Flushes the current wrapped block into the lines.
     fn flush_wrapping(&mut self) -> Result<()> {
-        if let Some(w) = self.wrapping.take() {
-            self.lines
-                .extend(w.into_lines()?.into_iter().map(RenderLine::Text))
+        if let Some(mut w) = self.wrapping.take() {
+            let frags = w.take_trailing_fragments();
+            self.extend_lines(w.into_lines()?.into_iter().map(RenderLine::Text));
+
+            self.pending_frags.extend(frags);
         }
         Ok(())
     }
@@ -1176,7 +1239,7 @@ impl<D: TextDecorator> SubRenderer<D> {
                                 buf = String::new();
                             }
 
-                            self.lines.push_back(RenderLine::Text(wrapped_line));
+                            self.add_line(RenderLine::Text(wrapped_line));
                             wrapped_line = TaggedLine::new();
                             pos = 0;
                         }
@@ -1192,7 +1255,7 @@ impl<D: TextDecorator> SubRenderer<D> {
                     pos += width;
                 }
             }
-            self.lines.push_back(RenderLine::Text(wrapped_line));
+            self.add_line(RenderLine::Text(wrapped_line));
         }
     }
 
@@ -1204,7 +1267,7 @@ impl<D: TextDecorator> SubRenderer<D> {
 
     fn add_horizontal_line(&mut self, line: BorderHoriz<Vec<D::Annotation>>) -> Result<()> {
         self.flush_wrapping()?;
-        self.lines.push_back(RenderLine::Line(line));
+        self.add_line(RenderLine::Line(line));
         Ok(())
     }
 
@@ -1238,7 +1301,7 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
     fn add_empty_line(&mut self) -> Result<()> {
         html_trace!("add_empty_line()");
         self.flush_all()?;
-        self.lines.push_back(RenderLine::Text(TaggedLine::new()));
+        self.add_line(RenderLine::Text(TaggedLine::new()));
         html_trace_quiet!("add_empty_line: at_block_end <- false");
         self.at_block_end = false;
         html_trace_quiet!("add_empty_line: new lines: {:?}", self.lines);
@@ -1286,7 +1349,7 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
 
     fn add_horizontal_border(&mut self) -> Result<()> {
         self.flush_wrapping()?;
-        self.lines.push_back(RenderLine::Line(BorderHoriz::new(
+        self.add_line(RenderLine::Line(BorderHoriz::new(
             self.width,
             self.ann_stack.clone(),
         )));
@@ -1295,7 +1358,7 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
 
     fn add_horizontal_border_width(&mut self, width: usize) -> Result<()> {
         self.flush_wrapping()?;
-        self.lines.push_back(RenderLine::Line(BorderHoriz::new(
+        self.add_line(RenderLine::Line(BorderHoriz::new(
             width,
             self.ann_stack.clone(),
         )));
@@ -1378,7 +1441,8 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
 
         self.flush_wrapping()?;
         let tag = self.ann_stack.clone();
-        self.lines.extend(
+
+        self.extend_lines(
             other
                 .into_lines()?
                 .into_iter()
@@ -1407,6 +1471,7 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
                     }
                 }),
         );
+
         Ok(())
     }
 
@@ -1545,11 +1610,11 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
                     );
                 }
             }
-            self.lines.push_back(RenderLine::Text(line));
+            self.add_line(RenderLine::Text(line));
             line = TaggedLine::new();
         }
         if self.options.draw_borders {
-            self.lines.push_back(RenderLine::Line(next_border));
+            self.add_line(RenderLine::Line(next_border));
         }
         Ok(())
     }


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
index 2580e85..ea10477 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -1,5 +1,6 @@
 use crate::config::Config;
-use crate::render::text_renderer::PlainDecorator;
+use crate::render::text_renderer::{PlainDecorator, TaggedString};
+use crate::render::TaggedLineElement;
 use crate::{config, Error};
 
 use super::render::text_renderer::{RichAnnotation, RichDecorator, TaggedLine, TrivialDecorator};
@@ -652,6 +653,7 @@ Hello
         20,
     );
 }
+
 #[test]
 fn test_link() {
     test_html(
@@ -1218,6 +1220,166 @@ fn test_pre_tab() {
     test_html(b"<pre>Helloooo\tworld</pre>", "Helloooo        world\n", 40);
 }
 
+#[test]
+fn test_pre_tab2() {
+    // Note hard tab characters below.
+    test_html(
+        br#"<pre>	t0
+x	t1
+xx	t2
+xxx	t3
+xxxx	t4
+xxxxx	t5
+xxxxxx	t6
+xxxxxxx	t7
+xxxxxxxx	t8
+xxxxxxxxx	t9</pre>"#,
+        r"        t0
+x       t1
+xx      t2
+xxx     t3
+xxxx    t4
+xxxxx   t5
+xxxxxx  t6
+xxxxxxx t7
+xxxxxxxx        t8
+xxxxxxxxx       t9
+",
+        40,
+    );
+}
+
+// Check for edge cases hitting the width
+#[test]
+fn test_pre_tab3() {
+    // Note hard tab characters below.
+    test_html(
+        br#"<pre>	t
+x	t
+xx	t
+xxx	t
+xxxx	t
+xxxxx	t
+xxxxxx	t
+xxxxxxx	t
+xxxxxxxx	t
+xxxxxxxxx	t</pre>"#,
+        r"        t
+x       t
+xx      t
+xxx     t
+xxxx    t
+xxxxx   t
+xxxxxx  t
+xxxxxxx t
+xxxxxxxx  
+t
+xxxxxxxxx 
+t
+",
+        10,
+    );
+    test_html(
+        br#"<pre>	t
+x	t
+xx	t
+xxx	t
+xxxx	t
+xxxxx	t
+xxxxxx	t
+xxxxxxx	t
+xxxxxxxx	t
+xxxxxxxxx	t</pre>"#,
+        r"        t
+x       t
+xx      t
+xxx     t
+xxxx    t
+xxxxx   t
+xxxxxx  t
+xxxxxxx t
+xxxxxxxx 
+t
+xxxxxxxxx
+        t
+",
+        9,
+    );
+    test_html(
+        br#"<pre>	t
+x	t
+xx	t
+xxx	t
+xxxx	t
+xxxxx	t
+xxxxxx	t
+xxxxxxx	t
+xxxxxxxx	t
+xxxxxxxxx	t</pre>"#,
+        r"        
+t
+x       
+t
+xx      
+t
+xxx     
+t
+xxxx    
+t
+xxxxx   
+t
+xxxxxx  
+t
+xxxxxxx 
+t
+xxxxxxxx
+        
+t
+xxxxxxxx
+x       
+t
+",
+        8,
+    );
+    test_html(
+        br#"<pre>	t
+x	t
+xx	t
+xxx	t
+xxxx	t
+xxxxx	t
+xxxxxx	t
+xxxxxxx	t
+xxxxxxxx	t
+xxxxxxxxx	t</pre>"#,
+        r"       
+t
+x      
+t
+xx     
+t
+xxx    
+t
+xxxx   
+t
+xxxxx  
+t
+xxxxxx 
+t
+xxxxxxx
+       
+t
+xxxxxxx
+x      
+t
+xxxxxxx
+xx     
+t
+",
+        7,
+    );
+}
+
 #[test]
 fn test_em_strong() {
     test_html(
@@ -2222,6 +2384,58 @@ fn test_issue_187() {
     let _ = crate::config::plain().string_from_read(&html[..], 17);
 }
 
+fn get_lines(html: &[u8], width: usize) -> Vec<TaggedLine<Vec<()>>> {
+    config::plain().lines_from_read(html, width).unwrap()
+}
+
+#[test]
+fn frag_simple() {
+    use TaggedLineElement::*;
+    assert_eq!(
+        get_lines(br#"<p id="my_id">Hi</p>"#, 10)
+            .into_iter()
+            .map(|line| line.into_iter().collect::<Vec<_>>())
+            .collect::<Vec<_>>(),
+        vec![vec![
+            FragmentStart("my_id".into()),
+            Str(TaggedString {
+                s: "Hi".into(),
+                tag: Default::default(),
+            })
+        ],]
+    );
+}
+
+#[test]
+fn frag_list() {
+    use TaggedLineElement::*;
+    assert_eq!(
+        get_lines(
+            br#"<ul id="my_id">
+            <li>One</li>
+            <li>Two</li>
+        </ul>"#,
+            10
+        )
+        .into_iter()
+        .map(|line| line.into_iter().collect::<Vec<_>>())
+        .collect::<Vec<_>>(),
+        vec![
+            vec![
+                FragmentStart("my_id".into()),
+                Str(TaggedString {
+                    s: "* One".into(),
+                    tag: Default::default(),
+                })
+            ],
+            vec![Str(TaggedString {
+                s: "* Two".into(),
+                tag: Default::default(),
+            })],
+        ]
+    );
+}
+
 #[cfg(feature = "css")]
 mod css_tests {
     use super::{