Repository: jugglerchris/rust-html2text
Pull request #99 (2023-12-21 20:31:22)

Improve selector handling


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 004d71d..d2626f3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,12 @@ Possible log types:
 - `[fixed]` for any bug fixes.
 - `[security]` to invite users to upgrade in case of vulnerabilities.
 
+### 0.8.0
+
+- [added] CSS: Support more extensive selectors
+- [changed] CSS handling defaults to off; use `Config::use_doc_css()`
+  or `Config::add_css` to use CSS.
+
 ### 0.7.1
 
 - [added] Now recognised CSS `display:none`
diff --git a/Cargo.toml b/Cargo.toml
index 4fed751..559cfbc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "html2text"
-version = "0.7.1"
+version = "0.8.0"
 authors = ["Chris Emerson <github@mail.nosreme.org>"]
 description = "Render HTML as plain text."
 repository = "https://github.com/jugglerchris/rust-html2text/"
diff --git a/src/css.rs b/src/css.rs
index 7f9eb4f..1bcee59 100644
--- a/src/css.rs
+++ b/src/css.rs
@@ -1,19 +1,179 @@
 //! Some basic CSS support.
-use std::{collections::HashMap, io::Write};
+use std::io::Write;
+use std::convert::TryFrom;
 use std::ops::Deref;
 
 use lightningcss::{stylesheet::{
     ParserOptions, StyleSheet
-}, rules::CssRule, properties::Property, values::color::CssColor};
+}, rules::CssRule, properties::{Property, display::{self, DisplayKeyword}}, values::color::CssColor};
 
 use crate::{Result, TreeMapResult, markup5ever_rcdom::{Handle, NodeData::{Comment, Document, Element, self}}, tree_map_reduce};
 
+#[derive(Debug, Clone)]
+enum SelectorComponent {
+    Class(String),
+    Element(String),
+    Star,
+    CombChild,
+    CombDescendant,
+}
+
+#[derive(Debug, Clone)]
+struct Selector {
+    // List of components, right first so we match from the leaf.
+    components: Vec<SelectorComponent>,
+}
+
+impl Selector {
+    fn do_matches(comps: &[SelectorComponent], node: &Handle) -> bool {
+        match comps.first() {
+            None => return true,
+            Some(comp) => {
+                match comp {
+                    SelectorComponent::Class(class) => {
+                        match &node.data {
+                            Document |
+                                NodeData::Doctype { .. } |
+                                NodeData::Text { .. } |
+                                Comment { .. } |
+                                NodeData::ProcessingInstruction { .. } => {
+                                    return false;
+                                }
+                            Element { attrs, .. } => {
+                                let attrs = attrs.borrow();
+                                for attr in attrs.iter() {
+                                    if &attr.name.local == "class" {
+                                        for cls in attr.value.split_whitespace() {
+                                            if cls == class {
+                                                return Self::do_matches(&comps[1..], node);
+                                            }
+                                        }
+                                    }
+                                }
+                                return false;
+                            }
+                        }
+                    }
+                    SelectorComponent::Element(name) => {
+                        match &node.data {
+                            Element { name: eltname, .. } => {
+                                if name == eltname.expanded().local.deref() {
+                                    return Self::do_matches(&comps[1..], node);
+                                } else {
+                                    return false;
+                                }
+                            }
+                            _ => {
+                                return false;
+                            }
+                        }
+                    }
+                    SelectorComponent::Star => {
+                        return Self::do_matches(&comps[1..], node);
+                    }
+                    SelectorComponent::CombChild => {
+                        if let Some(parent) = node.parent.take() {
+                            let parent_handle = parent.upgrade();
+                            node.parent.set(Some(parent));
+                            if let Some(ph) = parent_handle {
+                                return Self::do_matches(&comps[1..], &ph);
+                            } else {
+                                return false;
+                            }
+                        } else {
+                            return false;
+                        }
+                    }
+                    SelectorComponent::CombDescendant => {
+                        if let Some(parent) = node.parent.take() {
+                            let parent_handle = parent.upgrade();
+                            node.parent.set(Some(parent));
+                            if let Some(ph) = parent_handle {
+                                return Self::do_matches(&comps[1..], &ph) ||
+                                    Self::do_matches(comps, &ph);
+                            } else {
+                                return false;
+                            }
+                        } else {
+                            return false;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    fn matches(&self, node: &Handle) -> bool {
+        Self::do_matches(&self.components, node)
+    }
+}
+
+impl<'r, 'i> TryFrom<&'r lightningcss::selector::Selector<'i>> for Selector {
+    type Error = ();
+
+    fn try_from(selector: &'r lightningcss::selector::Selector<'i>) -> std::result::Result<Self, Self::Error> {
+        let mut components = Vec::new();
+
+        use lightningcss::selector::Component;
+        use lightningcss::selector::Combinator;
+
+        let mut si = selector.iter();
+        loop {
+            while let Some(item) = si.next() {
+                match item {
+                    Component::Class(id) => {
+                        components.push(SelectorComponent::Class(String::from(id.deref())));
+                    }
+                    Component::LocalName(name) => {
+                        components.push(SelectorComponent::Element(String::from(name.lower_name.deref())));
+                    }
+                    Component::ExplicitUniversalType => {
+                        components.push(SelectorComponent::Star);
+                    }
+                    _ => {
+                        html_trace!("Unknown component {:?}", item);
+                        return Err(());
+                    }
+                }
+            }
+            if let Some(comb) = si.next_sequence() {
+                match comb {
+                    Combinator::Child => {
+                        components.push(SelectorComponent::CombChild);
+                    }
+                    Combinator::Descendant => {
+                        components.push(SelectorComponent::CombDescendant);
+                    }
+                    _ => {
+                        html_trace!("Unknown combinator {:?}", comb);
+                        return Err(());
+                    }
+                }
+            } else {
+                break;
+            }
+        }
+        Ok(Selector {
+            components
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) enum Style {
+    Colour(CssColor),
+    DisplayNone,
+}
+
+#[derive(Debug, Clone)]
+struct Ruleset {
+    selector: Selector,
+    styles: Vec<Style>,
+}
+
 /// Stylesheet data which can be used while building the render tree.
 #[derive(Clone, Default, Debug)]
 pub struct StyleData {
-    /// Map from classes to colours
-    pub(crate) colours: HashMap<String, CssColor>,
-    pub(crate) display: HashMap<String, lightningcss::properties::display::Display>,
+    rules: Vec<Ruleset>,
 }
 
 impl StyleData {
@@ -21,39 +181,38 @@ impl StyleData {
     /// and the relevant and supported features extracted.
     pub fn add_css(&mut self, css: &str) {
         let ss = StyleSheet::parse(css, ParserOptions::default()).unwrap();
+        html_trace!("add css [[{}]]", css);
 
         for rule in &ss.rules.0 {
             match rule {
                 CssRule::Style(style) => {
+                    let mut styles = Vec::new();
                     for decl in &style.declarations.declarations {
                         match decl {
                             Property::Color(color) => {
-                                for selector in &style.selectors.0 {
-                                    for item in selector.iter() {
-                                        use lightningcss::selector::Component;
-                                        match item {
-                                            Component::Class(c) => { 
-                                                self.colours.insert(c.0.to_string(), color.clone());
-                                            }
-                                            _ => {  }
-                                        }
-                                    }
-                                }
+                                styles.push(Style::Colour(color.clone()));
                             }
                             Property::Display(disp) => {
-                                for selector in &style.selectors.0 {
-                                    for item in selector.iter() {
-                                        use lightningcss::selector::Component;
-                                        match item {
-                                            Component::Class(c) => { 
-                                                self.display.insert(c.0.to_string(), disp.clone());
-                                            }
-                                            _ => {  }
-                                        }
-                                    }
+                                if let display::Display::Keyword(DisplayKeyword::None) = disp {
+                                    styles.push(Style::DisplayNone);
+                                }
+                            }
+                            _ => {}
+                        }
+                    }
+                    if !styles.is_empty() {
+                        for selector in &style.selectors.0 {
+                            match Selector::try_from(selector) {
+                                Ok(selector) => {
+                                    self.rules.push(Ruleset {
+                                        selector,
+                                        styles: styles.clone()
+                                    });
+                                }
+                                Err(_) => {
+                                    continue;
                                 }
                             }
-                            _ => (),
                         }
                     }
                 }
@@ -65,8 +224,18 @@ impl StyleData {
     /// Merge style data from other into this one.
     /// Data on other takes precedence.
     pub fn merge(&mut self, other: Self) {
-        self.colours.extend(other.colours);
-        self.display.extend(other.display);
+        self.rules.extend(other.rules);
+    }
+
+    pub(crate) fn matching_rules(&self, handle: &Handle) -> Vec<Style> {
+        let mut result = Vec::new();
+        for rule in &self.rules {
+            if rule.selector.matches(handle) {
+                result.extend(rule.styles.iter().cloned());
+            }
+        }
+
+        result
     }
 }
 
diff --git a/src/lib.rs b/src/lib.rs
index d643ee4..3d8298c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -305,7 +305,7 @@ impl RenderTable {
     pub fn rows_mut(&mut self) -> std::slice::IterMut<RenderTableRow> {
         self.rows.iter_mut()
     }
-    /// Consume this and return a Vec<RenderNode> containing the children;
+    /// Consume this and return a `Vec<RenderNode>` containing the children;
     /// the children know the column sizes required.
     pub fn into_rows(self, col_sizes: Vec<usize>, vert: bool) -> Vec<RenderNode> {
         self.rows
@@ -393,15 +393,15 @@ pub enum RenderNodeInfo {
     Ol(i64, Vec<RenderNode>),
     /// A description list (containing Dt or Dd)
     Dl(Vec<RenderNode>),
-    /// A term (from a <dl>)
+    /// A term (from a `<dl>`)
     Dt(Vec<RenderNode>),
-    /// A definition (from a <dl>)
+    /// A definition (from a `<dl>`)
     Dd(Vec<RenderNode>),
     /// A line break
     Break,
     /// A table
     Table(RenderTable),
-    /// A set of table rows (from either <thead> or <tbody>
+    /// A set of table rows (from either `<thead>` or `<tbody>`
     TableBody(Vec<RenderTableRow>),
     /// Table row (must only appear within a table body)
     /// If the boolean is true, then the cells are drawn vertically
@@ -933,6 +933,8 @@ where
 struct HtmlContext {
     #[cfg(feature = "css")]
     style_data: css::StyleData,
+    #[cfg(feature = "css")]
+    use_doc_css: bool,
 }
 
 fn dom_to_render_tree_with_context<T: Write>(
@@ -942,7 +944,7 @@ fn dom_to_render_tree_with_context<T: Write>(
 -> Result<Option<RenderNode>> {
     html_trace!("### dom_to_render_tree: HTML: {:?}", handle);
     #[cfg(feature = "css")]
-    {
+    if context.use_doc_css {
         let mut doc_style_data = css::dom_to_stylesheet(handle.clone(), err_out)?;
         doc_style_data.merge(context.style_data);
         context.style_data = doc_style_data;
@@ -1054,31 +1056,20 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
             let mut frag_from_name_attr = false;
 
             #[cfg(feature = "css")]
-            let classes = {
-                let mut classes = Vec::new();
-                let borrowed = attrs.borrow();
-                for attr in borrowed.iter() {
-                    if &attr.name.local == "class" {
-                        for class in attr.value.split_whitespace() {
-                            classes.push(class.to_string());
-                        }
-                    }
-                }
-                classes
-            };
+            let mut css_colour = None;
             #[cfg(feature = "css")]
-            for class in &classes {
-                if let Some(disp) = context.style_data.display.get(class) {
-                    use lightningcss::properties::display;
-                    match disp {
-                        display::Display::Keyword(display::DisplayKeyword::None) => {
-                            // Hide display: none
+            {
+                for style in context.style_data.matching_rules(&handle) {
+                    match style {
+                        css::Style::Colour(col) => {
+                            css_colour = Some(col);
+                        }
+                        css::Style::DisplayNone => {
                             return Ok(Nothing);
                         }
-                        _ => {}
                     }
                 }
-            }
+            };
             let result = match name.expanded() {
                 expanded_name!(html "html")
                 | expanded_name!(html "body") => {
@@ -1102,14 +1093,7 @@ fn process_dom_node<'a, 'b, 'c, T: Write>(
                     }
                     #[cfg(feature = "css")]
                     {
-                        let mut colour = None;
-                        for class in classes {
-                            if let Some(c) = context.style_data.colours.get(&class) {
-                                colour = Some(c);
-                                break;
-                            }
-                        }
-                        if let Some(Ok(colour)) = colour.map(TryFrom::try_from) {
+                        if let Some(Ok(colour)) = css_colour.as_ref().map(TryFrom::try_from) {
                             pending(handle, move |_, cs| Ok(Some(RenderNode::new(Coloured(colour, vec![RenderNode::new(Container(cs))])))))
                         } else {
                             /* process children, but don't add anything */
@@ -1731,6 +1715,8 @@ pub mod config {
 
         #[cfg(feature = "css")]
         style: StyleData,
+        #[cfg(feature = "css")]
+        use_doc_css: bool,
     }
 
     impl<D: TextDecorator> Config<D> {
@@ -1741,6 +1727,8 @@ pub mod config {
                 HtmlContext {
                     #[cfg(feature = "css")]
                     style_data: std::mem::take(&mut self.style),
+                    #[cfg(feature = "css")]
+                    use_doc_css: self.use_doc_css,
                 })
         }
 
@@ -1766,6 +1754,15 @@ pub mod config {
             self.style.add_css(css);
             self
         }
+
+        #[cfg(feature = "css")]
+        /// Parse CSS from any <style> elements and use supported rules.
+        pub fn use_doc_css(mut self) -> Self {
+            {
+                self.use_doc_css = true;
+            }
+            self
+        }
     }
 
     impl Config<RichDecorator> {
@@ -1805,7 +1802,9 @@ pub mod config {
         Config {
             decorator: RichDecorator::new(),
             #[cfg(feature = "css")]
-            style: Default::default()
+            style: Default::default(),
+            #[cfg(feature = "css")]
+            use_doc_css: false,
         }
     }
 
@@ -1814,7 +1813,9 @@ pub mod config {
         Config {
             decorator: PlainDecorator::new(),
             #[cfg(feature = "css")]
-            style: Default::default()
+            style: Default::default(),
+            #[cfg(feature = "css")]
+            use_doc_css: false,
         }
     }
 
@@ -1823,7 +1824,9 @@ pub mod config {
         Config {
             decorator,
             #[cfg(feature = "css")]
-            style: Default::default()
+            style: Default::default(),
+            #[cfg(feature = "css")]
+            use_doc_css: false,
         }
     }
 }
diff --git a/src/render/text_renderer.rs b/src/render/text_renderer.rs
index dc8af32..76c1c2f 100644
--- a/src/render/text_renderer.rs
+++ b/src/render/text_renderer.rs
@@ -813,7 +813,7 @@ pub struct SubRenderer<D: TextDecorator> {
     decorator: D,
     ann_stack: Vec<D::Annotation>,
     text_filter_stack: Vec<fn(&str) -> Option<String>>,
-    /// The depth of <pre> block stacking.
+    /// The depth of `<pre>` block stacking.
     pre_depth: usize,
 }
 


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
index df1b491..914b4cd 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -13,7 +13,15 @@ macro_rules! assert_eq_str {
     };
 }
 fn test_html(input: &[u8], expected: &str, width: usize) {
-    assert_eq_str!(from_read(input, width), expected);
+    let output = from_read(input, width);
+    assert_eq_str!(output, expected);
+}
+#[cfg(feature = "css")]
+fn test_html_css(input: &[u8], expected: &str, width: usize) {
+    let result = config::plain()
+        .use_doc_css()
+        .string_from_read(input, width).unwrap();
+    assert_eq_str!(result, expected);
 }
 fn test_html_err(input: &[u8], expected: Error, width: usize) {
     let result = config::plain()
@@ -1615,28 +1623,79 @@ fn test_issue_93_x() {
 }
 
 #[cfg(feature = "css")]
-#[test]
-fn test_disp_none() {
-    test_html(br#"
-      <style>
-          .hide { display: none; }
-      </style>
-    <p>Hello</p>
-    <p class="hide">Ignore</p>
-    <p>There</p>"#,
-    r#"Hello
+mod css_tests {
+    use super::{test_html_css, test_html_style};
+
+    #[test]
+    fn test_disp_none() {
+        test_html_css(br#"
+          <style>
+              .hide { display: none; }
+          </style>
+        <p>Hello</p>
+        <p class="hide">Ignore</p>
+        <p>There</p>"#,
+        r#"Hello
 
 There
 "#, 20);
 
-    // Same as above, but style supplied separately.
-    test_html_style(br#"
-    <p>Hello</p>
-    <p class="hide">Ignore</p>
-    <p>There</p>"#,
-    " .hide { display: none; }",
-    r#"Hello
+        // Same as above, but style supplied separately.
+        test_html_style(br#"
+        <p>Hello</p>
+        <p class="hide">Ignore</p>
+        <p>There</p>"#,
+        " .hide { display: none; }",
+        r#"Hello
 
 There
 "#, 20);
+    }
+
+    #[test]
+    fn test_selector_elementname()
+    {
+        test_html_css(br#"
+          <style>
+              div { display: none; }
+          </style>
+        <p>Hello</p>
+        <div>Ignore</div>
+        <p>There</p>"#,
+        r#"Hello
+
+There
+"#, 20);
+    }
+
+    #[test]
+    fn test_selector_aoc()
+    {
+        test_html_css(br#"
+          <style>
+              .someclass > * > span > span {
+                  display: none;
+              }
+          </style>
+        <p>Hello</p>
+        <div class="someclass">Ok
+        <p>
+         <span>Span1<span>Span2</span></span>
+        </p>
+        <div>
+         <span>Span1<span>Span2</span></span>
+        </div>
+        </div>
+        <p>There</p>"#,
+        r#"Hello
+
+Ok
+
+Span1
+
+Span1
+
+There
+"#, 20);
+    }
 }