Repository: jugglerchris/rust-html2text
Pull request #105 (2023-12-29 17:25:25)

Add methods to access DOM and render trees.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c818795..b3a5561 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,14 @@ Possible log types:
 - `[fixed]` for any bug fixes.
 - `[security]` to invite users to upgrade in case of vulnerabilities.
 
+### 0.9.2
+
+- [fixed] CSS didn't work inside `<ul>` or `<ol>`.
+- [added] Add methods to get and use the intermediate HTML DOM and RenderTree
+  from Config.
+- [fixed] Removed some clones which are no longer necessary now that Box<FnOnce>
+  works.
+
 ### 0.9.1
 
 - [fixed] Various documentation issues (thanks sgtatham)
diff --git a/Cargo.toml b/Cargo.toml
index cf66240..3ccb18d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "html2text"
-version = "0.9.1"
+version = "0.9.2"
 authors = ["Chris Emerson <github@mail.nosreme.org>"]
 description = "Render HTML as plain text."
 repository = "https://github.com/jugglerchris/rust-html2text/"
diff --git a/src/lib.rs b/src/lib.rs
index 3f12ced..a935442 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -127,7 +127,7 @@ impl TryFrom<&CssColor> for Colour {
 }
 
 /// Errors from reading or rendering HTML
-#[derive(thiserror::Error, Debug, Eq, PartialEq)]
+#[derive(thiserror::Error, Debug)]
 #[non_exhaustive]
 pub enum Error {
     /// The output width was too narrow to render to.
@@ -143,8 +143,27 @@ pub enum Error {
     /// A formatting error happened
     #[error("Formatting error")]
     FmtError(#[from] std::fmt::Error),
+    /// An I/O error
+    #[error("I/O error")]
+    IoError(#[from] std::io::Error),
 }
 
+impl PartialEq for Error {
+    fn eq(&self, other: &Error) -> bool {
+        use Error::*;
+        match (self, other) {
+            (TooNarrow, TooNarrow) => true,
+            #[cfg(feature = "css")]
+            (CssParseError, CssParseError) => true,
+            (Fail, Fail) => true,
+            (FmtError(f1), FmtError(f2)) => f1 == f2,
+            _ => false,
+        }
+    }
+}
+
+impl Eq for Error {}
+
 type Result<T> = std::result::Result<T, Error>;
 
 /// A dummy writer which does nothing
@@ -422,6 +441,8 @@ pub enum RenderNodeInfo {
     FragStart(String),
     /// A region with classes
     Coloured(Colour, Vec<RenderNode>),
+    /// A list item
+    ListItem(Vec<RenderNode>),
 }
 
 /// Common fields from a node.
@@ -481,7 +502,7 @@ impl RenderNode {
 
             Container(ref v) | Em(ref v) | Strong(ref v) | Strikeout(ref v) | Code(ref v)
             | Block(ref v) | Div(ref v) | Pre(ref v) | BlockQuote(ref v) | Dl(ref v)
-            | Dt(ref v) | Dd(ref v) => v
+            | Dt(ref v) | Dd(ref v) | ListItem(ref v) => v
                 .iter()
                 .map(RenderNode::get_size_estimate)
                 .fold(Default::default(), SizeEstimate::add),
@@ -553,6 +574,7 @@ impl RenderNode {
             | Strikeout(ref v)
             | Code(ref v)
             | Block(ref v)
+            | ListItem(ref v)
             | Div(ref v)
             | Pre(ref v)
             | BlockQuote(ref v)
@@ -589,6 +611,7 @@ fn precalc_size_estimate<'a>(node: &'a RenderNode) -> Result<TreeMapResult<(), &
         | Strikeout(ref v)
         | Code(ref v)
         | Block(ref v)
+        | ListItem(ref v)
         | Div(ref v)
         | Pre(ref v)
         | BlockQuote(ref v)
@@ -638,42 +661,21 @@ fn precalc_size_estimate<'a>(node: &'a RenderNode) -> Result<TreeMapResult<(), &
 }
 
 /// Make a Vec of RenderNodes from the children of a node.
-fn children_to_render_nodes<T: Write>(handle: Handle, err_out: &mut T) -> Result<Vec<RenderNode>> {
+fn children_to_render_nodes<T: Write>(handle: Handle, err_out: &mut T, context: &mut HtmlContext) -> Result<Vec<RenderNode>> {
     /* process children, but don't add anything */
     handle
         .children
         .borrow()
         .iter()
-        .flat_map(|ch| dom_to_render_tree(ch.clone(), err_out).transpose())
+        .flat_map(|ch| dom_to_render_tree_with_context(ch.clone(), err_out, context).transpose())
         .collect()
 }
 
-/// Make a Vec of RenderNodes from the <li> children of a node.
-fn list_children_to_render_nodes<T: Write>(handle: Handle, err_out: &mut T) -> Result<Vec<RenderNode>> {
-    let mut children = Vec::new();
-
-    for child in handle.children.borrow().iter() {
-        match child.data {
-            Element { ref name, .. } => match name.expanded() {
-                expanded_name!(html "li") => {
-                    let li_children = children_to_render_nodes(child.clone(), err_out)?;
-                    children.push(RenderNode::new(RenderNodeInfo::Block(li_children)));
-                }
-                _ => {}
-            },
-            Comment { .. } => {}
-            _ => {
-                html_trace!("Unhandled in list: {:?}\n", child);
-            }
-        }
-    }
-    Ok(children)
-}
-
-/// Make a Vec of DtElements from the <dt> and <dd> children of a node.
+/// Make a Vec of DtElements from the `<dt>` and `<dd>` children of a node.
 fn desc_list_children_to_render_nodes<T: Write>(
     handle: Handle,
     err_out: &mut T,
+    context: &mut HtmlContext,
 ) -> Result<Vec<RenderNode>> {
     let mut children = Vec::new();
 
@@ -681,11 +683,11 @@ fn desc_list_children_to_render_nodes<T: Write>(
         match child.data {
             Element { ref name, .. } => match name.expanded() {
                 expanded_name!(html "dt") => {
-                    let dt_children = children_to_render_nodes(child.clone(), err_out)?;
+                    let dt_children = children_to_render_nodes(child.clone(), err_out, context)?;
                     children.push(RenderNode::new(RenderNodeInfo::Dt(dt_children)));
                 }
                 expanded_name!(html "dd") => {
-                    let dd_children = children_to_render_nodes(child.clone(), err_out)?;
+                    let dd_children = children_to_render_nodes(child.clone(), err_out, context)?;
                     children.push(RenderNode::new(RenderNodeInfo::Dd(dd_children)));
                 }
                 _ => {}
@@ -826,7 +828,7 @@ fn td_to_render_tree<'a, 'b, T: Write>(
 /// A reducer which combines results from mapping children into
 /// the result for the current node.  Takes a context and a
 /// vector of results and returns a new result (or nothing).
-type ResultReducer<'a, C, R> = dyn Fn(&mut C, Vec<R>) -> Result<Option<R>> + 'a;
+type ResultReducer<'a, C, R> = dyn FnOnce(&mut C, Vec<R>) -> Result<Option<R>> + 'a;
 
 /// A closure to call before processing a child node.
 type ChildPreFn<C, N> = dyn Fn(&mut C, &N) -> Result<()>;
@@ -1002,6 +1004,7 @@ fn prepend_marker(prefix: RenderNode, mut orig: RenderNode) -> RenderNode {
         // For Container, we do the same thing just to make the data
         // less pointlessly nested.
         Block(ref mut children)
+        | ListItem(ref mut children)
         | Div(ref mut children)
         | Pre(ref mut children)
         | BlockQuote(ref mut children)
@@ -1136,15 +1139,10 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                     PendingChildren {
                         children: handle.children.borrow().clone(),
                         cons: if let Some(href) = target {
-                            // We need the closure to own the string it's going to use.
-                            // Unfortunately that means we ideally want FnOnce; but
-                            // that doesn't yet work in a Box.  Box<FnBox()> does, but
-                            // is unstable.  So we'll just move a string in and clone
-                            // it on use.
                             let href: String = href.into();
                             Box::new(move |_, cs: Vec<RenderNode>| {
                                 if cs.iter().any(|c| !c.is_shallow_empty()) {
-                                    Ok(Some(RenderNode::new(Link(href.clone(), cs))))
+                                    Ok(Some(RenderNode::new(Link(href, cs))))
                                 } else {
                                     Ok(None)
                                 }
@@ -1199,6 +1197,9 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                 expanded_name!(html "p") => {
                     pending(handle, |_, cs| Ok(Some(RenderNode::new(Block(cs)))))
                 }
+                expanded_name!(html "li") => {
+                    pending(handle, |_, cs| Ok(Some(RenderNode::new(ListItem(cs)))))
+                }
                 expanded_name!(html "div") => {
                     pending(handle, |_, cs| Ok(Some(RenderNode::new(Div(cs)))))
                 }
@@ -1217,9 +1218,9 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                 expanded_name!(html "blockquote") => {
                     pending(handle, |_, cs| Ok(Some(RenderNode::new(BlockQuote(cs)))))
                 }
-                expanded_name!(html "ul") => Finished(RenderNode::new(Ul(
-                    list_children_to_render_nodes(handle.clone(), err_out)?,
-                ))),
+                expanded_name!(html "ul") => {
+                    pending(handle, |_, cs| Ok(Some(RenderNode::new(Ul(cs)))))
+                }
                 expanded_name!(html "ol") => {
                     let borrowed = attrs.borrow();
                     let mut start = 1;
@@ -1230,13 +1231,20 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                         }
                     }
 
-                    Finished(RenderNode::new(Ol(
-                        start,
-                        list_children_to_render_nodes(handle.clone(), err_out)?,
-                    )))
+                    pending(handle, move |_, cs| {
+                        dbg!(&cs);
+                        let cs = cs.into_iter()
+                            .filter(|n| match &n.info {
+                                RenderNodeInfo::ListItem(..) => true,
+                                _ => false,
+                            })
+                            .collect();
+                        Ok(Some(RenderNode::new(Ol(start, dbg!(cs)))))
+                    })
+
                 }
                 expanded_name!(html "dl") => Finished(RenderNode::new(Dl(
-                    desc_list_children_to_render_nodes(handle.clone(), err_out)?,
+                    desc_list_children_to_render_nodes(handle.clone(), err_out, context)?,
                 ))),
                 _ => {
                     html_trace!("Unhandled element: {:?}\n", name.local);
@@ -1272,7 +1280,7 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                             prefn,
                             postfn,
                             cons: Box::new(move |ctx, ch| {
-                                let fragnode = RenderNode::new(FragStart(fragname.clone()));
+                                let fragnode = RenderNode::new(FragStart(fragname));
                                 match cons(ctx, ch)? {
                                     None => Ok(Some(fragnode)),
                                     Some(node) => Ok(Some(prepend_marker(fragnode, node))),
@@ -1297,7 +1305,16 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                             cons: Box::new(move |ctx, ch| {
                                 Ok(cons(ctx, ch)?
                                     .map(|n| {
-                                        RenderNode::new(Coloured(colour.into(), vec![n]))
+                                        // Special case: lists need to recognise ListItem nodes as
+                                        // direct children.
+                                        match n.info {
+                                            ListItem(children) => {
+                                                let coloured = RenderNode::new(Coloured(colour.into(), children));
+                                                RenderNode::new(
+                                                    ListItem(vec![coloured]))
+                                            }
+                                            ni => RenderNode::new(Coloured(colour.into(), vec![RenderNode::new(ni)])),
+                                        }
                                     }))
                             }),
                         }
@@ -1413,7 +1430,7 @@ fn do_render_node<'a, 'b, T: Write, D: TextDecorator>(
             renderer.add_image(&src, &title)?;
             Finished(None)
         }
-        Block(children) => {
+        Block(children) | ListItem(children) => {
             renderer.start_block()?;
             pending2(children, |renderer: &mut TextRenderer<D>, _| {
                 renderer.end_block();
@@ -1755,6 +1772,7 @@ pub mod config {
     }, Result, RenderTree, HtmlContext};
     #[cfg(feature = "css")]
     use crate::css::StyleData;
+    use super::{Discard, Error};
 
     /// Configure the HTML processing.
     pub struct Config<D: TextDecorator> {
@@ -1770,10 +1788,10 @@ pub mod config {
 
     impl<D: TextDecorator> Config<D> {
         /// Make the HtmlContext from self.
-        fn make_context(&mut self) -> HtmlContext {
+        fn make_context(&self) -> HtmlContext {
             HtmlContext {
                 #[cfg(feature = "css")]
-                style_data: std::mem::take(&mut self.style),
+                style_data: self.style.clone(),
                 #[cfg(feature = "css")]
                 use_doc_css: self.use_doc_css,
 
@@ -1787,6 +1805,47 @@ pub mod config {
                 context)
         }
 
+        /// Parse the HTML into a DOM structure.
+        pub fn parse_html<R: std::io::Read>(&self, mut input: R) -> Result<super::RcDom> {
+            use html5ever::tendril::TendrilSink;
+            let opts = super::ParseOpts {
+                tree_builder: super::TreeBuilderOpts {
+                    drop_doctype: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            };
+            Ok(super::parse_document(super::RcDom::default(), opts)
+                .from_utf8()
+                .read_from(&mut input)?)
+        }
+
+        /// Convert an HTML DOM into a RenderTree.
+        pub fn dom_to_render_tree(&self, dom: &super::RcDom) -> Result<RenderTree> {
+            Ok(RenderTree(super::dom_to_render_tree_with_context(
+                    dom.document.clone(),
+                    &mut Discard {},
+                    &mut self.make_context())?
+                    .ok_or(Error::Fail)?))
+        }
+
+        /// Render an existing RenderTree into a string.
+        pub fn render_to_string(&self, render_tree: RenderTree, width: usize) -> Result<String> {
+            Ok(render_tree.render_with_context(
+                &mut self.make_context(), width, self.decorator.make_subblock_decorator())?
+                .into_string()?)
+        }
+
+        /// Take an existing RenderTree, and returns text wrapped to `width` columns.
+        /// The text is returned as a `Vec<TaggedLine<_>>`; the annotations are vectors
+        /// of the provided text decorator's `Annotation`.  The "outer" annotation comes first in
+        /// the `Vec`.
+        pub fn render_to_lines(&self, render_tree: RenderTree, width: usize) -> Result<Vec<TaggedLine<Vec<D::Annotation>>>> {
+            Ok(render_tree.render_with_context(
+                &mut self.make_context(), width, self.decorator.make_subblock_decorator())?
+                .into_lines()?)
+        }
+
         /// Reads HTML from `input`, and returns a `String` with text wrapped to
         /// `width` columns.
         pub fn string_from_read<R: std::io::Read>(mut self, input: R, width: usize) -> Result<String> {
@@ -1862,6 +1921,32 @@ pub mod config {
             }
             Ok(result)
         }
+
+        /// Return coloured text from a RenderTree.  `colour_map` is a function which takes a list
+        /// of `RichAnnotation` and some text, and returns the text with any terminal escapes
+        /// desired to indicate those annotations (such as colour).
+        pub fn render_coloured<FMap>(
+            &self,
+            render_tree: RenderTree,
+            width: usize,
+            colour_map: FMap,
+        ) -> Result<String>
+        where
+            FMap: Fn(&[RichAnnotation], &str) -> String,
+        {
+            use std::fmt::Write;
+
+            let lines = self.render_to_lines(render_tree, width)?;
+
+            let mut result = String::new();
+            for line in lines {
+                for ts in line.tagged_strings() {
+                    write!(result, "{}", colour_map(&ts.tag, &ts.s))?;
+                }
+                result.push('\n');
+            }
+            Ok(result)
+        }
     }
 
     /// Return a Config initialized with a `RichDecorator`.


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
index 059c1dd..924be5c 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -1837,4 +1837,64 @@ There
         r#"Test <R>red</R>
 "#, 20);
     }
+
+    #[test]
+    fn test_css_lists()
+    {
+        test_html_coloured(br##"
+          <style>
+              .red {
+                  color:#FF0000;
+              }
+          </style>
+        <ul>
+          <li class="red">Line one</li>
+          <li>Line <span class="red">two</span></li>
+        </ul>
+        "##,
+        r#"* <R>Line one</R>
+* Line <R>two</R>
+"#, 20);
+        test_html_coloured(br##"
+          <style>
+              .red {
+                  color:#FF0000;
+              }
+          </style>
+        <ol>
+          <li class="red">Line one</li>
+          <li>Line <span class="red">two</span></li>
+        </ul>
+        "##,
+        r#"1. <R>Line one</R>
+2. Line <R>two</R>
+"#, 20);
+    }
+
+    #[test]
+    fn test_coloured_multi()
+    {
+        use super::test_colour_map;
+        let config = crate::config::rich()
+                    .use_doc_css();
+        let dom = config.parse_html(&br##"
+          <style>
+              .red {
+                  color:#FF0000;
+              }
+          </style>
+        <p>Test paragraph with <span class="red">red</span> text</p>
+        "##[..]).unwrap();
+        let rt = config.dom_to_render_tree(&dom).unwrap();
+        assert_eq!(config.render_coloured(rt.clone(), 10, test_colour_map).unwrap(),
+        r#"Test
+paragraph
+with <R>red</R>
+text
+"#);
+        assert_eq!(config.render_coloured(rt.clone(), 100, test_colour_map).unwrap(),
+        r#"Test paragraph with <R>red</R> text
+"#);
+    }
+
 }