Repository: mitsuhiko/minijinja
Pull request #653 (2024-12-02 07:36:35)

Fixes incorrect wordcount result when the string includes unicode marks
The `is_alphanumeric` function does not correctly handle Unicode marks, which leads to incorrect results in some cases. For example, the input `{{ helloाworld|wordcount }}` incorrectly returns `1`, whereas the correct result should be `2`.

Intentionally skipped updating the changelog as version 2.6.0 has not been released yet.


Patch:
diff --git a/Cargo.lock b/Cargo.lock
index b779986..e22725a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1976,6 +1976,7 @@ dependencies = [
  "textwrap",
  "time",
  "time-tz",
+ "unicode_categories",
 ]
 
 [[package]]
@@ -3372,6 +3373,12 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
 [[package]]
 name = "unindent"
 version = "0.2.3"
diff --git a/minijinja-cli/Cargo.toml b/minijinja-cli/Cargo.toml
index d6521e4..89b60f1 100644
--- a/minijinja-cli/Cargo.toml
+++ b/minijinja-cli/Cargo.toml
@@ -46,7 +46,7 @@ minijinja = { version = "=2.5.0", path = "../minijinja", features = [
     "custom_syntax",
     "loop_controls"
 ] }
-minijinja-contrib = { version = "=2.5.0", optional = true, path = "../minijinja-contrib", features = ["pycompat", "datetime", "timezone", "rand", "unicode_wordwrap"] }
+minijinja-contrib = { version = "=2.5.0", optional = true, path = "../minijinja-contrib", features = ["pycompat", "datetime", "timezone", "rand", "unicode_wordwrap", "wordcount"] }
 rustyline = { version = "14.0.0", optional = true }
 serde = { version = "1.0.183", features = ["derive", "rc"] }
 serde_json = "1.0.105"
diff --git a/minijinja-contrib/Cargo.toml b/minijinja-contrib/Cargo.toml
index 556614f..4de9250 100644
--- a/minijinja-contrib/Cargo.toml
+++ b/minijinja-contrib/Cargo.toml
@@ -21,6 +21,7 @@ pycompat = ["minijinja/builtins"]
 datetime = ["time"]
 timezone = ["time-tz"]
 rand = ["dep:rand"]
+wordcount = ["unicode_categories"]
 wordwrap = ["textwrap"]
 unicode_wordwrap = ["wordwrap", "textwrap/unicode-linebreak", "textwrap/unicode-width"]
 
@@ -31,6 +32,7 @@ serde = "1.0.164"
 textwrap = { version = "0.16.1", optional = true, default-features = false, features = ["smawk"] }
 time = { version = "0.3.35", optional = true, features = ["serde", "formatting", "parsing"] }
 time-tz = { version = "1.0.3", features = ["db"], optional = true }
+unicode_categories = { version = "0.1.1", optional = true }
 
 [dev-dependencies]
 insta = { version = "1.38.0", features = ["glob", "serde"] }
diff --git a/minijinja-contrib/src/filters/mod.rs b/minijinja-contrib/src/filters/mod.rs
index 9cdf001..257c088 100644
--- a/minijinja-contrib/src/filters/mod.rs
+++ b/minijinja-contrib/src/filters/mod.rs
@@ -220,14 +220,18 @@ pub fn truncate(state: &State, value: Value, kwargs: Kwargs) -> Result<String, E
 /// ```jinja
 /// {{ "Hello world!"|wordcount }}
 /// ```
+#[cfg(feature = "wordcount")]
+#[cfg_attr(docsrs, doc(cfg(feature = "wordcount")))]
 pub fn wordcount(value: Value) -> Result<Value, Error> {
+    use unicode_categories::UnicodeCategories;
+
     let s = value.as_str().unwrap_or_default();
     let mut count: u32 = 0;
     let mut in_word = false;
 
     // Iterate through characters, counting transitions from non-word to word chars
     for c in s.chars() {
-        let is_word_char = c.is_alphanumeric() || c == '_';
+        let is_word_char = c.is_letter() || c.is_numeric() || c == '_';
         if is_word_char && !in_word {
             count += 1;
             in_word = true;
diff --git a/minijinja-contrib/src/lib.rs b/minijinja-contrib/src/lib.rs
index e56ddbc..d1a9e11 100644
--- a/minijinja-contrib/src/lib.rs
+++ b/minijinja-contrib/src/lib.rs
@@ -35,7 +35,10 @@ pub fn add_to_environment(env: &mut Environment) {
     env.add_filter("pluralize", filters::pluralize);
     env.add_filter("filesizeformat", filters::filesizeformat);
     env.add_filter("truncate", filters::truncate);
-    env.add_filter("wordcount", filters::wordcount);
+    #[cfg(feature = "wordcount")]
+    {
+        env.add_filter("wordcount", filters::wordcount);
+    }
     #[cfg(feature = "wordwrap")]
     {
         env.add_filter("wordwrap", filters::wordwrap);


Test patch:
diff --git a/minijinja-contrib/tests/filters.rs b/minijinja-contrib/tests/filters.rs
index 3649b8d..82d10af 100644
--- a/minijinja-contrib/tests/filters.rs
+++ b/minijinja-contrib/tests/filters.rs
@@ -1,5 +1,5 @@
 use minijinja::{context, Environment};
-use minijinja_contrib::filters::{pluralize, wordcount};
+use minijinja_contrib::filters::pluralize;
 use similar_asserts::assert_eq;
 
 #[test]
@@ -204,7 +204,10 @@ fn test_truncate() {
 }
 
 #[test]
+#[cfg(feature = "wordwrap")]
 fn test_wordcount() {
+    use minijinja_contrib::filters::wordcount;
+
     let mut env = Environment::new();
     env.add_filter("wordcount", wordcount);
 
@@ -266,6 +269,18 @@ fn test_wordcount() {
         .unwrap(),
         "3"
     );
+
+    // Test unicode marks
+    assert_eq!(
+        env.render_str(
+            "{{ text|wordcount }}",
+            context! {
+                text => "helloाworld"
+            }
+        )
+        .unwrap(),
+        "2"
+    );
 }
 
 #[test]