Repository: mitsuhiko/minijinja
Pull request #728 (2025-02-18 19:02:41)

Parser: fix implicit concat for allocated strings
String literals should behave the same regardless of whether they contain escape sequences. 
This PR fixes the string literal "implicit concat" rule to account for `Token::String`s.

mitsuhiko: Good catch!

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a8696ac..2b575d3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,8 @@ All notable changes to MiniJinja are documented here.
 - Fixed an issue with undeclared variables not handling `caller`.  #725
 - Removed unnecessary `Filters` and `Tests` traits.  They remain as
   hidden aliases to `Function`.  #726
+- Fixed a bug that caused implicit string concatenation to not correctly
+  handle escapes.  #728
 
 ## 2.8.0
 
diff --git a/minijinja/src/compiler/parser.rs b/minijinja/src/compiler/parser.rs
index cbdac0a..2ce3c15 100644
--- a/minijinja/src/compiler/parser.rs
+++ b/minijinja/src/compiler/parser.rs
@@ -638,19 +638,30 @@ impl<'a> Parser<'a> {
             Token::Ident("false" | "False") => Ok(const_val!(false)),
             Token::Ident("none" | "None") => Ok(const_val!(())),
             Token::Ident(name) => Ok(ast::Expr::Var(Spanned::new(ast::Var { id: name }, span))),
-            Token::Str(val) => {
-                if matches_token!(self, Token::Str(_)) {
-                    let mut buf = String::from(val);
-                    while let Some((Token::Str(s), _)) = ok!(self.stream.current()) {
-                        buf.push_str(s);
-                        ok!(self.stream.next());
+            Token::Str(val)
+                if !matches!(
+                    self.stream.current(),
+                    Ok(Some((Token::Str(_), _) | (Token::String(_), _)))
+                ) =>
+            {
+                Ok(const_val!(val))
+            }
+            Token::Str(_) | Token::String(_) => {
+                let mut buf = match token {
+                    Token::Str(s) => s.to_owned(),
+                    Token::String(s) => s,
+                    _ => unreachable!(),
+                };
+                loop {
+                    match ok!(self.stream.current()) {
+                        Some((Token::Str(s), _)) => buf.push_str(s),
+                        Some((Token::String(s), _)) => buf.push_str(s),
+                        _ => break,
                     }
-                    Ok(const_val!(buf))
-                } else {
-                    Ok(const_val!(val))
+                    ok!(self.stream.next());
                 }
+                Ok(const_val!(buf))
             }
-            Token::String(val) => Ok(const_val!(val)),
             Token::Int(val) => Ok(const_val!(val)),
             Token::Int128(val) => Ok(const_val!(val)),
             Token::Float(val) => Ok(const_val!(val)),


Test patch:
diff --git a/minijinja/tests/parser-inputs/string-implicit-concat.txt b/minijinja/tests/parser-inputs/string-implicit-concat.txt
index a0c6285..75ca0d7 100644
--- a/minijinja/tests/parser-inputs/string-implicit-concat.txt
+++ b/minijinja/tests/parser-inputs/string-implicit-concat.txt
@@ -1,3 +1,6 @@
 {{ "foo" }}
 {{ "foo" "bar" }}
 {{ "foo" "bar" "baz" }}
+{{ "foo" "\u2603bar" "baz" }}
+{{ "foo\u2603bar" "baz" }}
+{{ "foo\nbar" "bar\nbaz" }}
diff --git a/minijinja/tests/snapshots/test_parser__parser@string-implicit-concat.txt.snap b/minijinja/tests/snapshots/test_parser__parser@string-implicit-concat.txt.snap
index 231267f..c9ca450 100644
--- a/minijinja/tests/snapshots/test_parser__parser@string-implicit-concat.txt.snap
+++ b/minijinja/tests/snapshots/test_parser__parser@string-implicit-concat.txt.snap
@@ -1,6 +1,6 @@
 ---
 source: minijinja/tests/test_parser.rs
-description: "{{ \"foo\" }}\n{{ \"foo\" \"bar\" }}\n{{ \"foo\" \"bar\" \"baz\" }}"
+description: "{{ \"foo\" }}\n{{ \"foo\" \"bar\" }}\n{{ \"foo\" \"bar\" \"baz\" }}\n{{ \"foo\" \"\\u2603bar\" \"baz\" }}\n{{ \"foo\\u2603bar\" \"baz\" }}\n{{ \"foo\\nbar\" \"bar\\nbaz\" }}"
 input_file: minijinja/tests/parser-inputs/string-implicit-concat.txt
 ---
 Ok(
@@ -27,6 +27,30 @@ Ok(
                     value: "foobarbaz",
                 } @ 3:3-3:20,
             } @ 3:0-3:20,
+            EmitRaw {
+                raw: "\n",
+            } @ 3:23-4:0,
+            EmitExpr {
+                expr: Const {
+                    value: "foo☃barbaz",
+                } @ 4:3-4:26,
+            } @ 4:0-4:26,
+            EmitRaw {
+                raw: "\n",
+            } @ 4:29-5:0,
+            EmitExpr {
+                expr: Const {
+                    value: "foo☃barbaz",
+                } @ 5:3-5:23,
+            } @ 5:0-5:23,
+            EmitRaw {
+                raw: "\n",
+            } @ 5:26-6:0,
+            EmitExpr {
+                expr: Const {
+                    value: "foo\nbarbar\nbaz",
+                } @ 6:3-6:24,
+            } @ 6:0-6:24,
         ],
-    } @ 0:0-3:23,
+    } @ 0:0-6:27,
 )