Repository: zhiburt/tabled
Pull request #430 (2024-10-16 00:29:45)

Upgrade unicode-width and handle width more correctly
Current versions of unicode-width have more handling for characters and
ligatures that typically render at a different width than a
character-by-character width sum would produce, such as ZWJ emoji
sequences.

Current versions of unicode-width also punt on control characters (as
somewhat recommended by Unicode standards), making the assumption that
if not specially handled they might render as width-1 substitution
characters. unicode-width recommends filtering them out and handling
them specially if that isn't the desired behavior.

Update all the dimension-checking functions to always check whole lines
at a time, splitting on newlines, and filtering out all other control
characters. This makes most tests pass.

Remove one test that appears to have depended on the exact rendering
width of Devanagari, which unicode-width notes is not consistent or
feasible, and doesn't have consistent monospace handling.

Add a test for emoji width handling, including a ZWJ-based emoji
cluster that should render as a single emoji.

Note that terminals are inconsistent about how they render ZWJ-based
emoji clusters. There's a proposal "mode 2027" to help applications tell
the terminal how they want to handle such sequences. In the future,
tabled/papergrid *might* want to add support for both modes, so that
applications built on tabled/papergrid can then choose which handling
they want (either based on user configurability or terminal
detection/configuration based on "mode 2027"). See
https://mitchellh.com/writing/grapheme-clusters-in-terminals for
details.

In the meantime, this implements the ideal behavior of assuming
ZWJ-based emoji clusters are rendered as a single emoji, and will
produce incorrect widths on terminals that render such emoji clusters
incorrectly. See the table at
https://mitchellh.com/writing/grapheme-clusters-in-terminals#terminal-comparison
for a comparison of terminal behavior.

----------

If you want the behavior of handling emoji clusters as multiple emoji (which will match the behavior of the terminals that do so), and ignoring ligatures, I *think* that behavior would correspond to `get_string_width` summing the widths of each character individually. However, I've sent a feature request to unicode-width upstream to request a function that gives the exact behavior.

I haven't made any attempt here to add configurability to tabled to allow choosing between those two behaviors. That'd be a much larger change.

In any case, this change may be sufficiently disruptive that you might want to publish it as a new semver-major version. It's more correct according to Unicode, and should produce better results for some people and somewhat less incorrect results for others (it shouldn't be *worse* for anyone), but it's a substantial behavior change that people may want to knowingly adopt.


Patch:
diff --git a/papergrid/Cargo.toml b/papergrid/Cargo.toml
index 3050b3e..c6100a0 100644
--- a/papergrid/Cargo.toml
+++ b/papergrid/Cargo.toml
@@ -13,7 +13,7 @@ std = []
 ansi = ["ansi-str", "ansitok"]
 
 [dependencies]
-unicode-width = "=0.1.11"
+unicode-width = "0.2"
 bytecount = "0.6"
 fnv = "1.0"
 ansi-str = { version = "0.8", optional = true }
diff --git a/papergrid/src/util/string.rs b/papergrid/src/util/string.rs
index 2fa863b..8eada2f 100644
--- a/papergrid/src/util/string.rs
+++ b/papergrid/src/util/string.rs
@@ -7,33 +7,16 @@
 /// Returns string width and count lines of a string. It's a combination of [`string_width_multiline`] and [`count_lines`].
 #[cfg(feature = "std")]
 pub fn get_text_dimension(text: &str) -> (usize, usize) {
-    #[cfg(not(feature = "ansi"))]
-    {
-        let (lines, acc, max) = text.chars().fold((1, 0, 0), |(lines, acc, max), c| {
-            if c == '\n' {
-                (lines + 1, 0, acc.max(max))
-            } else {
-                let w = unicode_width::UnicodeWidthChar::width(c).unwrap_or(0);
-                (lines, acc + w, max)
-            }
-        });
-
-        (lines, acc.max(max))
-    }
-
-    #[cfg(feature = "ansi")]
-    {
-        get_lines(text)
-            .map(|line| get_line_width(&line))
-            .fold((0, 0), |(i, acc), width| (i + 1, acc.max(width)))
-    }
+    get_lines(text)
+        .map(|line| get_line_width(&line))
+        .fold((0, 0), |(i, acc), width| (i + 1, acc.max(width)))
 }
 
 /// Returns a string width.
 pub fn get_line_width(text: &str) -> usize {
     #[cfg(not(feature = "ansi"))]
     {
-        unicode_width::UnicodeWidthStr::width(text)
+        get_string_width(text)
     }
 
     #[cfg(feature = "ansi")]
@@ -44,7 +27,7 @@ pub fn get_line_width(text: &str) -> usize {
         ansitok::parse_ansi(text)
             .filter(|e| e.kind() == ansitok::ElementKind::Text)
             .map(|e| &text[e.start()..e.end()])
-            .map(unicode_width::UnicodeWidthStr::width)
+            .map(get_string_width)
             .sum()
     }
 }
@@ -53,10 +36,7 @@ pub fn get_line_width(text: &str) -> usize {
 pub fn get_text_width(text: &str) -> usize {
     #[cfg(not(feature = "ansi"))]
     {
-        text.lines()
-            .map(unicode_width::UnicodeWidthStr::width)
-            .max()
-            .unwrap_or(0)
+        text.lines().map(get_string_width).max().unwrap_or(0)
     }
 
     #[cfg(feature = "ansi")]
@@ -72,7 +52,7 @@ pub fn get_char_width(c: char) -> usize {
 
 /// Returns a string width (accouting all characters).
 pub fn get_string_width(text: &str) -> usize {
-    unicode_width::UnicodeWidthStr::width(text)
+    unicode_width::UnicodeWidthStr::width(text.replace(|c| c < ' ', "").as_str())
 }
 
 /// Calculates a number of lines.


Test patch:
diff --git a/papergrid/tests/grid/render.rs b/papergrid/tests/grid/render.rs
index 6fcd8f3..21a4327 100644
--- a/papergrid/tests/grid/render.rs
+++ b/papergrid/tests/grid/render.rs
@@ -207,14 +207,32 @@ test_table!(
     "+----+--+"
 );
 
+#[test]
+fn emoji_width_test() {
+    use papergrid::util::string::get_string_width;
+    assert_eq!(get_string_width("👩"), 2);
+    assert_eq!(get_string_width("🔬"), 2);
+    assert_eq!(get_string_width("👩\u{200D}🔬"), 2);
+}
+
+test_table!(
+    emoji_handling,
+    grid(2, 1).data([["👩👩👩👩👩👩"], ["Hello"]]).build(),
+    "+------------+"
+    "|👩👩👩👩👩👩|"
+    "+------------+"
+    "|Hello       |"
+    "+------------+"
+);
+
 test_table!(
-    hieroglyph_handling_2,
-    grid(2, 1).data([["জী._ডি._ব্লক_সল্টলেক_দূর্গা_পুজো_২০১৮.jpg"], ["Hello"]]).build(),
-    "+------------------------------------+"
-    "|জী._ডি._ব্লক_সল্টলেক_দূর্গা_পুজো_২০১৮.jpg|"
-    "+------------------------------------+"
-    "|Hello                               |"
-    "+------------------------------------+"
+    emoji_handling_2,
+    grid(2, 1).data([["👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬"], ["Hello"]]).build(),
+    "+------------+"
+    "|👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬👩\u{200D}🔬|"
+    "+------------+"
+    "|Hello       |"
+    "+------------+"
 );
 
 test_table!(
diff --git a/testing_table/Cargo.toml b/testing_table/Cargo.toml
index f9310b6..1aedcb9 100644
--- a/testing_table/Cargo.toml
+++ b/testing_table/Cargo.toml
@@ -17,4 +17,4 @@ ansi = ["ansitok"]
 
 [dependencies]
 ansitok = { version = "0.2", optional = true }
-unicode-width = "=0.1.11"
+unicode-width = "0.2"