Repository: mitsuhiko/minijinja (language: rust)

Issue:
raw/endraw includes raw/endraw tags in the output
## Description

The tags are included in the output, when I don't believe that they should be.

## Reproduction steps

Modify the hello example to enclose Hello in a raw block:

```diff
diff --git a/examples/hello/src/main.rs b/examples/hello/src/main.rs
index 05119d1..d488681 100644
--- a/examples/hello/src/main.rs
+++ b/examples/hello/src/main.rs
@@ -2,7 +2,7 @@ use minijinja::{context, Environment};

 fn main() {
     let mut env = Environment::new();
-    env.add_template("hello.txt", "Hello {{ name }}!").unwrap();
+    env.add_template("hello.txt", "{% raw %}Hello{% endraw %} {{ name }}!").unwrap();
     let template = env.get_template("hello.txt").unwrap();
     println!("{}", template.render(context!(name => "John")).unwrap());
 }
```

Running it produces this output:

```
{% raw %}Hello{% endraw %} John!
```

Additional helpful information:

- Version of minijinja: 0.30, 0.34 and 1.0 (`main` as of rev 95eb8448d9c4ac706a8c35ad040ee9aac7e1cd3c)
- Version of rustc: `rustc 1.70.0 (90c541806 2023-05-31)`
- Operating system and version: Linux Fedora 38

## What did you expect

I'd expect this output:

```
Hello John!
```

https://jbmoelker.github.io/jinja-compat-tests/tags/raw/ shows some various examples of the raw tag/block syntax in other implementations for the sake of comparison.



Discussion / hints:
Hah yeah. That seems very wrong.

Fix patch:
diff --git a/minijinja/src/compiler/lexer.rs b/minijinja/src/compiler/lexer.rs
index 4ca9b4b9..e4caa5a1 100644
--- a/minijinja/src/compiler/lexer.rs
+++ b/minijinja/src/compiler/lexer.rs
@@ -401,20 +401,28 @@ pub fn tokenize(
                             // raw blocks require some special handling.  If we are at the beginning of a raw
                             // block we want to skip everything until {% endraw %} completely ignoring iterior
                             // syntax and emit the entire raw block as TemplateData.
-                            if let Some((mut ptr, _)) =
+                            if let Some((raw, trim_start)) =
                                 skip_basic_tag(&state.rest[skip..], "raw", block_end)
                             {
-                                ptr += skip;
+                                state.advance(raw + skip);
+                                let mut ptr = 0;
                                 while let Some(block) =
                                     memstr(&state.rest.as_bytes()[ptr..], block_start.as_bytes())
                                 {
                                     ptr += block + block_start.len();
-                                    if let Some((endraw, trim)) =
+                                    let trim_end = state.rest.as_bytes().get(ptr) == Some(&b'-');
+                                    if let Some((endraw, trim_next)) =
                                         skip_basic_tag(&state.rest[ptr..], "endraw", block_end)
                                     {
-                                        let result = &state.rest[..ptr + endraw];
+                                        let mut result = &state.rest[..ptr - block_start.len()];
+                                        if trim_start {
+                                            result = result.trim_start();
+                                        }
+                                        if trim_end {
+                                            result = result.trim_end();
+                                        }
                                         state.advance(ptr + endraw);
-                                        trim_leading_whitespace = trim;
+                                        trim_leading_whitespace = trim_next;
                                         return Some(Ok((
                                             Token::TemplateData(result),
                                             state.span(old_loc),


Test patch:
diff --git a/minijinja/tests/snapshots/test_lexer__lexer@raw.txt.snap b/minijinja/tests/snapshots/test_lexer__lexer@raw.txt.snap
index a71ea351..a4808fe0 100644
--- a/minijinja/tests/snapshots/test_lexer__lexer@raw.txt.snap
+++ b/minijinja/tests/snapshots/test_lexer__lexer@raw.txt.snap
@@ -1,19 +1,18 @@
 ---
 source: minijinja/tests/test_lexer.rs
 description: "before {% raw %} this is a {{ raw }} {% block %} {% endraw %} after\n\nbefore {%- raw %} this is a {{ raw }} {% block %} {% endraw -%} after\n\nbefore {%- raw -%} this is a {{ raw }} {% block %} {%- endraw -%} after"
-input_file: minijinja/tests/lexer-inputs/raw.txt
 ---
 TemplateData("before ")
   "before "
-TemplateData("{% raw %} this is a {{ raw }} {% block %} {% endraw %}")
+TemplateData(" this is a {{ raw }} {% block %} ")
   "{% raw %} this is a {{ raw }} {% block %} {% endraw %}"
 TemplateData(" after\n\nbefore")
   " after\n\nbefore"
-TemplateData("{%- raw %} this is a {{ raw }} {% block %} {% endraw -%}")
+TemplateData(" this is a {{ raw }} {% block %} ")
   "{%- raw %} this is a {{ raw }} {% block %} {% endraw -%}"
 TemplateData("after\n\nbefore")
   "after\n\nbefore"
-TemplateData("{%- raw -%} this is a {{ raw }} {% block %} {%- endraw -%}")
+TemplateData("this is a {{ raw }} {% block %}")
   "{%- raw -%} this is a {{ raw }} {% block %} {%- endraw -%}"
 TemplateData("after")
   "after"