Repository: jugglerchris/rust-html2text
Pull request #201 (2025-02-09 16:10:46)

Make footnote output configurable independently of the decorator.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f16f91..f1844a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,16 @@ Possible log types:
 - [changed] Various small refactors (thanks sftse)
 - [changed] `Config::rich()` no longer includes decorations around `<em>` etc. - 
   use `Config::rich().do_decorate()` to get the old behaviour.
+- [fixed] Remove unnecessary empty lines at the start of lists (thanks russellbanks)
+- [added] New CSS support: `::before`/`::after` and `content: "string"`, which is now
+  used for simple decorations.  With CSS enabled, this allows for customising
+  the display of `<em>foo</em>` without writing a decorator.
+- [added] Add support for `<h5>` and `<h6>` (thanks noahbaculi)
+- [changed] Link footnotes are now configurable independently of the decorator, and on
+  by default for `config::plain()` but can be enabled or disabled with
+  `config.link_footnotes(true/false)`.  The footnote references (e.g. `[1]`) are added
+  in the main renderer, and the actual footnotes are written in a default implementation
+  of `TextDecorator::finalise()` so can be customised.
 
 ### 0.13.6
 
diff --git a/examples/html2text.rs b/examples/html2text.rs
index b69c288..5144822 100644
--- a/examples/html2text.rs
+++ b/examples/html2text.rs
@@ -99,6 +99,15 @@ fn update_config<T: TextDecorator>(mut config: Config<T>, flags: &Flags) -> Conf
     if flags.use_css {
         config = config.use_doc_css();
     }
+    match (flags.link_footnotes, flags.no_link_footnotes) {
+        (true, true) => {
+            eprintln!("Error: can't specify both --link-footnotes and --no-link-footnotes");
+            std::process::exit(1);
+        }
+        (true, false) => config = config.link_footnotes(true),
+        (false, true) => config = config.link_footnotes(false),
+        (false, false) => {}
+    };
     config
 }
 
@@ -173,6 +182,8 @@ struct Flags {
     show_render: bool,
     #[cfg(feature = "css")]
     show_css: bool,
+    link_footnotes: bool,
+    no_link_footnotes: bool,
 }
 
 fn main() {
@@ -195,6 +206,8 @@ fn main() {
         show_render: false,
         #[cfg(feature = "css")]
         show_css: false,
+        link_footnotes: false,
+        no_link_footnotes: false,
     };
     let mut literal: bool = false;
 
@@ -225,6 +238,16 @@ fn main() {
             StoreTrue,
             "Output only literal text (no decorations)",
         );
+        ap.refer(&mut flags.link_footnotes).add_option(
+            &["--link-footnotes"],
+            StoreTrue,
+            "Enable link footnotes",
+        );
+        ap.refer(&mut flags.no_link_footnotes).add_option(
+            &["--no-link-footnotes"],
+            StoreTrue,
+            "Enable link footnotes",
+        );
         #[cfg(unix)]
         ap.refer(&mut flags.use_colour).add_option(
             &["--colour"],
diff --git a/src/lib.rs b/src/lib.rs
index 0cb7572..a77e895 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -291,7 +291,7 @@ pub(crate) struct ComputedStyle {
 
 impl ComputedStyle {
     /// Return the style data inherited by children.
-    pub(crate) fn inherit(self: &Self) -> Self {
+    pub(crate) fn inherit(&self) -> Self {
         // TODO: clear fields that shouldn't be inherited
         self.clone()
     }
@@ -1290,6 +1290,7 @@ struct HtmlContext {
     raw: bool,
     draw_borders: bool,
     wrap_links: bool,
+    include_link_footnotes: bool,
 }
 
 // Input to render tree conversion.
@@ -1513,7 +1514,7 @@ fn process_dom_node<T: Write>(
                 let computed =
                     context
                         .style_data
-                        .computed_style(&*parent_style, _handle, use_doc_css);
+                        .computed_style(parent_style, _handle, use_doc_css);
                 #[cfg(feature = "css")]
                 match computed.display.val() {
                     Some(css::Display::None) => return Ok(Nothing),
@@ -1741,7 +1742,7 @@ fn process_dom_node<T: Write>(
             let result = if computed_before.is_some() || computed_after.is_some() {
                 let wrap_nodes = move |mut node: RenderNode| {
                     if let Some(ref content) = computed_before {
-                        if let Some(ref pseudo_content) = content.content.val() {
+                        if let Some(pseudo_content) = content.content.val() {
                             node = insert_child(
                                 RenderNode::new(Text(pseudo_content.text.clone())),
                                 node,
@@ -1750,7 +1751,7 @@ fn process_dom_node<T: Write>(
                         }
                     }
                     if let Some(ref content) = computed_after {
-                        if let Some(ref pseudo_content) = content.content.val() {
+                        if let Some(pseudo_content) = content.content.val() {
                             node = insert_child(
                                 RenderNode::new(Text(pseudo_content.text.clone())),
                                 node,
@@ -1888,17 +1889,17 @@ impl PushedStyleInfo {
         let mut result: PushedStyleInfo = Default::default();
         #[cfg(feature = "css")]
         if let Some(col) = style.colour.val() {
-            render.push_colour(col.clone());
+            render.push_colour(*col);
             result.colour = true;
         }
         #[cfg(feature = "css")]
         if let Some(col) = style.bg_colour.val() {
-            render.push_bgcolour(col.clone());
+            render.push_bgcolour(*col);
             result.bgcolour = true;
         }
         if let Some(ws) = style.white_space.val() {
             if let WhiteSpace::Pre | WhiteSpace::PreWrap = ws {
-                render.push_ws(ws.clone());
+                render.push_ws(*ws);
                 result.white_space = true;
             }
         }
@@ -2410,6 +2411,7 @@ pub mod config {
         raw: bool,
         draw_borders: bool,
         wrap_links: bool,
+        include_link_footnotes: bool,
     }
 
     impl<D: TextDecorator> Config<D> {
@@ -2427,6 +2429,7 @@ pub mod config {
                 raw: self.raw,
                 draw_borders: self.draw_borders,
                 wrap_links: self.wrap_links,
+                include_link_footnotes: self.include_link_footnotes,
             }
         }
         /// Parse with context.
@@ -2637,6 +2640,12 @@ pub mod config {
             ]);
             self
         }
+
+        /// Add footnotes for hyperlinks
+        pub fn link_footnotes(mut self, include_footnotes: bool) -> Self {
+            self.include_link_footnotes = include_footnotes;
+            self
+        }
     }
 
     impl Config<RichDecorator> {
@@ -2686,7 +2695,9 @@ pub mod config {
 
     /// Return a Config initialized with a `PlainDecorator`.
     pub fn plain() -> Config<PlainDecorator> {
-        with_decorator(PlainDecorator::new()).do_decorate()
+        with_decorator(PlainDecorator::new())
+            .do_decorate()
+            .link_footnotes(true)
     }
 
     /// Return a Config initialized with a `PlainDecorator`.
@@ -2708,6 +2719,7 @@ pub mod config {
             raw: false,
             draw_borders: true,
             wrap_links: true,
+            include_link_footnotes: false,
         }
     }
 }
@@ -2744,6 +2756,7 @@ impl RenderTree {
             raw: context.raw,
             draw_borders: context.draw_borders,
             wrap_links: context.wrap_links,
+            include_link_footnotes: context.include_link_footnotes,
         };
         let test_decorator = decorator.make_subblock_decorator();
         let builder = SubRenderer::new(width, render_options, decorator);
diff --git a/src/markup5ever_rcdom.rs b/src/markup5ever_rcdom.rs
index ef3ea24..4b7a759 100644
--- a/src/markup5ever_rcdom.rs
+++ b/src/markup5ever_rcdom.rs
@@ -349,10 +349,10 @@ impl TreeSink for RcDom {
     }
 
     fn elem_name<'a>(&self, target: &'a Handle) -> ExpandedName<'a> {
-        return match target.data {
+        match target.data {
             NodeData::Element { ref name, .. } => name.expanded(),
             _ => panic!("not an element!"),
-        };
+        }
     }
 
     fn create_element(&self, name: QualName, attrs: Vec<Attribute>, flags: ElementFlags) -> Handle {
diff --git a/src/render/text_renderer.rs b/src/render/text_renderer.rs
index 9f2ae95..21d139c 100644
--- a/src/render/text_renderer.rs
+++ b/src/render/text_renderer.rs
@@ -9,11 +9,9 @@ use crate::WhiteSpace;
 use super::Renderer;
 use super::Result;
 use super::TooNarrow;
-use std::cell::Cell;
 use std::mem;
 use std::ops::Deref;
 use std::ops::DerefMut;
-use std::rc::Rc;
 use std::vec;
 use std::{collections::LinkedList, fmt::Debug};
 use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
@@ -59,6 +57,16 @@ impl<D: TextDecorator> TextRenderer<D> {
         Ok(())
     }
 
+    pub fn end_link(&mut self) -> Result<()> {
+        self.subrender.last_mut().unwrap().end_link()?;
+
+        if self.options.include_link_footnotes {
+            let footnote_num = self.links.len();
+            self.add_inline_text(&format!("[{}]", footnote_num))?;
+        }
+        Ok(())
+    }
+
     /// Push a new builder onto the stack
     pub fn push(&mut self, builder: SubRenderer<D>) {
         self.subrender.push(builder);
@@ -506,8 +514,8 @@ impl<T: Clone + Eq + Debug + Default> WrappedBlock<T> {
         Ok(())
     }
 
-    /// Consume self and return a vector of lines.
     /*
+    /// Consume self and return a vector of lines.
     pub fn into_untagged_lines(mut self) -> Vec<String> {
         self.flush();
 
@@ -757,9 +765,17 @@ pub trait TextDecorator {
         "}".into()
     }
 
-    /// Finish with a document, and return extra lines (eg footnotes)
-    /// to add to the rendered text.
-    fn finalise(&mut self, links: Vec<String>) -> Vec<TaggedLine<Self::Annotation>>;
+    /// Finish with a document, and return extra lines to add to the rendered text.
+    /// The urls are in the correct order for footnotes; if footnote references were
+    /// not included then this list will be empty.
+    fn finalise(&mut self, urls: Vec<String>) -> Vec<TaggedLine<Self::Annotation>> {
+        urls.into_iter()
+            .enumerate()
+            .map(|(idx, s)| {
+                TaggedLine::from_string(format!("[{}]: {}", idx + 1, s), &Default::default())
+            })
+            .collect()
+    }
 }
 
 /// A space on a horizontal row.
@@ -1029,6 +1045,9 @@ pub(crate) struct RenderOptions {
 
     /// Whether to wrap links as normal text
     pub wrap_links: bool,
+
+    /// Whether to include footnotes for hyperlinks
+    pub include_link_footnotes: bool,
 }
 
 impl Default for RenderOptions {
@@ -1040,6 +1059,7 @@ impl Default for RenderOptions {
             raw: false,
             draw_borders: true,
             wrap_links: true,
+            include_link_footnotes: false,
         }
     }
 }
@@ -1065,7 +1085,11 @@ fn get_wrapping_or_insert<'w, D: TextDecorator>(
 impl<D: TextDecorator> SubRenderer<D> {
     /// Render links as lines
     pub fn finalise(&mut self, links: Vec<String>) -> Vec<TaggedLine<D::Annotation>> {
-        self.decorator.finalise(links)
+        if self.options.include_link_footnotes {
+            self.decorator.finalise(links)
+        } else {
+            self.decorator.finalise(Vec::new())
+        }
     }
 
     /// Construct a new empty SubRenderer.
@@ -1706,17 +1730,13 @@ impl<D: TextDecorator> Renderer for SubRenderer<D> {
 /// A decorator for use with `SubRenderer` which outputs plain UTF-8 text
 /// with no annotations.  Markup is rendered as text characters or footnotes.
 #[derive(Clone, Debug)]
-pub struct PlainDecorator {
-    nlinks: Rc<Cell<usize>>,
-}
+pub struct PlainDecorator {}
 
 impl PlainDecorator {
     /// Create a new `PlainDecorator`.
     #[allow(clippy::new_without_default)]
     pub fn new() -> PlainDecorator {
-        PlainDecorator {
-            nlinks: Rc::new(Cell::new(0)),
-        }
+        PlainDecorator {}
     }
 }
 
@@ -1724,12 +1744,11 @@ impl TextDecorator for PlainDecorator {
     type Annotation = ();
 
     fn decorate_link_start(&mut self, _url: &str) -> (String, Self::Annotation) {
-        self.nlinks.set(self.nlinks.get() + 1);
         ("[".to_string(), ())
     }
 
     fn decorate_link_end(&mut self) -> String {
-        format!("][{}]", self.nlinks.get())
+        "]".to_string()
     }
 
     fn decorate_em_start(&self) -> (String, Self::Annotation) {
@@ -1787,14 +1806,6 @@ impl TextDecorator for PlainDecorator {
         format!("{}. ", i)
     }
 
-    fn finalise(&mut self, links: Vec<String>) -> Vec<TaggedLine<()>> {
-        links
-            .into_iter()
-            .enumerate()
-            .map(|(idx, s)| TaggedLine::from_string(format!("[{}]: {}", idx + 1, s), &()))
-            .collect()
-    }
-
     fn make_subblock_decorator(&self) -> Self {
         self.clone()
     }
@@ -1880,10 +1891,6 @@ impl TextDecorator for TrivialDecorator {
         "".to_string()
     }
 
-    fn finalise(&mut self, _links: Vec<String>) -> Vec<TaggedLine<()>> {
-        Vec::new()
-    }
-
     fn make_subblock_decorator(&self) -> Self {
         TrivialDecorator::new()
     }
@@ -2001,10 +2008,6 @@ impl TextDecorator for RichDecorator {
         format!("{}. ", i)
     }
 
-    fn finalise(&mut self, _links: Vec<String>) -> Vec<TaggedLine<RichAnnotation>> {
-        Vec::new()
-    }
-
     fn make_subblock_decorator(&self) -> Self {
         self.clone()
     }


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
index 1e5cbd4..2580e85 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -2,9 +2,7 @@ use crate::config::Config;
 use crate::render::text_renderer::PlainDecorator;
 use crate::{config, Error};
 
-#[cfg(feature = "css")]
-use super::render::text_renderer::RichDecorator;
-use super::render::text_renderer::{RichAnnotation, TaggedLine, TrivialDecorator};
+use super::render::text_renderer::{RichAnnotation, RichDecorator, TaggedLine, TrivialDecorator};
 use super::{from_read, from_read_with_decorator, parse, TextDecorator};
 
 /// Like assert_eq!(), but prints out the results normally as well
@@ -707,6 +705,84 @@ ple.com/
     );
 }
 
+#[test]
+fn test_links_footnotes() {
+    // Default plain includes footnotes
+    test_html(
+        br#"
+       <p>Hello, <a href="http://www.example.com/">world</a></p>"#,
+        r"Hello, [world][1]
+
+[1]: http://www.example.com/
+",
+        80,
+    );
+
+    // Can disable footnotes
+    test_html_conf(
+        br#"
+       <p>Hello, <a href="http://www.example.com/">world</a></p>"#,
+        r"Hello, [world]
+",
+        80,
+        |conf| conf.link_footnotes(false),
+    );
+}
+
+#[test]
+fn test_links_footnotes_trivial() {
+    // Trivial decorate does footnotes if enabled
+    test_html_conf_dec(
+        TrivialDecorator::new(),
+        br#"
+       <p>Hello, <a href="http://www.example.com/">world</a></p>"#,
+        r"Hello, world[1]
+
+[1]: http://www.example.com/
+",
+        80,
+        |conf| conf.link_footnotes(true),
+    );
+
+    // But by default doesn't
+    test_html_conf_dec(
+        TrivialDecorator::new(),
+        br#"
+       <p>Hello, <a href="http://www.example.com/">world</a></p>"#,
+        r"Hello, world
+",
+        80,
+        |conf| conf,
+    );
+}
+
+#[test]
+fn test_links_footnotes_rich() {
+    // Rich decorator  does include footnotes if enabled
+    test_html_conf_dec(
+        RichDecorator::new(),
+        br#"
+       <p>Hello, <a href="http://www.example.com/">world</a></p>"#,
+        r"Hello, world[1]
+
+[1]: http://www.example.com/
+",
+        80,
+        |conf| conf.link_footnotes(true),
+    );
+
+    // But by default doesn't
+    test_html_conf_dec(
+        RichDecorator::new(),
+        br#"
+       <p>Hello, <a href="http://www.example.com/">world</a></p>"#,
+        r"Hello, world
+",
+        80,
+        |conf| conf,
+    );
+}
+
 #[test]
 fn test_wrap() {
     test_html(