Repository: mitsuhiko/minijinja (rust)

Issue:
Overflow compiler/lexer.rs:382:22
## Description

I upgraded to the last version (2.8.0), never had a problem before and began to have overflow panics. 

I downgraded to 2.7.0 and there seems to be no issue in that version.

## Reproduction steps

Don't have a reproduction step for the moment

Additional helpful information:

- Version of minijinja: 2.8.0
- Version of rustc: 1.85
- Operating system and version: Arch 6.13.5

## What did you expect

Unexpected overflow


```rust
thread '<unnamed>' panicked at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/lexer.rs:382:22:
attempt to add with overflow
stack backtrace:
   0: rust_begin_unwind
             at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5
   1: core::panicking::panic_fmt
             at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14
   2: core::panicking::panic_const::panic_const_add_overflow
             at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:178:21
   3: minijinja::compiler::lexer::Tokenizer::advance
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/lexer.rs:382:22
   4: minijinja::compiler::lexer::Tokenizer::tokenize_root
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/lexer.rs:623:30
   5: minijinja::compiler::lexer::Tokenizer::next_token
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/lexer.rs:348:47
   6: minijinja::compiler::parser::TokenStream::new
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/parser.rs:113:23
   7: minijinja::compiler::parser::Parser::new
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/parser.rs:249:21
   8: minijinja::compiler::parser::parse
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/compiler/parser.rs:1273:5
   9: minijinja::template::CompiledTemplate::_new_impl
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/template.rs:375:23
  10: minijinja::template::CompiledTemplate::new
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/template.rs:367:33
  11: minijinja::environment::Environment::template_from_named_str
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/environment.rs:408:53
  12: minijinja::environment::Environment::template_from_str
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/environment.rs:421:9
  13: minijinja::environment::Environment::render_str
             at /home/sam/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/minijinja-2.8.0/src/environment.rs:466:13

```



Pull request:
Prevent overflows to happen from very large templates
Fixes #740

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e372a8e..90430e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to MiniJinja are documented here.
 
+## 2.9.0
+
+- Do not panic if too large templates (too many lines or too many
+  columns) are loaded.  The error reporting will be wrong in those
+  cases but the templates will load.  #742
+
 ## 2.8.0
 
 - Added `SemiStrict` undefined mode that is like strict but allows
diff --git a/minijinja/src/compiler/lexer.rs b/minijinja/src/compiler/lexer.rs
index 35dc4c2..c0132b0 100644
--- a/minijinja/src/compiler/lexer.rs
+++ b/minijinja/src/compiler/lexer.rs
@@ -376,10 +376,10 @@ impl<'s> Tokenizer<'s> {
         for c in skipped.chars() {
             match c {
                 '\n' => {
-                    self.current_line += 1;
+                    self.current_line = self.current_line.saturating_add(1);
                     self.current_col = 0;
                 }
-                _ => self.current_col += 1,
+                _ => self.current_col = self.current_col.saturating_add(1),
             }
         }
         self.current_offset += bytes;
diff --git a/minijinja/src/compiler/tokens.rs b/minijinja/src/compiler/tokens.rs
index 27ef68b..94de7c8 100644
--- a/minijinja/src/compiler/tokens.rs
+++ b/minijinja/src/compiler/tokens.rs
@@ -137,10 +137,21 @@ pub struct Span {
 
 impl fmt::Debug for Span {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            " @ {}:{}-{}:{}",
-            self.start_line, self.start_col, self.end_line, self.end_col
-        )
+        if f.alternate() {
+            f.debug_struct("Span")
+                .field("start_line", &self.start_line)
+                .field("start_col", &self.start_col)
+                .field("start_offset", &self.start_offset)
+                .field("end_line", &self.end_line)
+                .field("end_col", &self.end_col)
+                .field("end_offset", &self.end_offset)
+                .finish()
+        } else {
+            write!(
+                f,
+                " @ {}:{}-{}:{}",
+                self.start_line, self.start_col, self.end_line, self.end_col
+            )
+        }
     }
 }


Test patch:
diff --git a/minijinja/tests/test_lexer.rs b/minijinja/tests/test_lexer.rs
index 97d799b..f7ad44a 100644
--- a/minijinja/tests/test_lexer.rs
+++ b/minijinja/tests/test_lexer.rs
@@ -108,3 +108,60 @@ fn test_trim_blocks() {
       "%}"
     "###);
 }
+
+#[test]
+fn test_overflowing_column() {
+    // Test that the lexer can handle a template with a very large column number.  This is
+    // important because the lexer uses u16 for line and column numbers.  If the column
+    // number overflows, it should saturate instead of panicking.
+    let mut input = String::with_capacity(70005);
+    for _ in 0..70000 {
+        input.push(' ');
+    }
+    input.push_str("{{ x }}");
+
+    let tokens_result: Result<Vec<_>, _> =
+        tokenize(&input, false, Default::default(), Default::default()).collect();
+    let spans = tokens_result
+        .unwrap()
+        .into_iter()
+        .map(|x| x.1)
+        .collect::<Vec<_>>();
+
+    insta::assert_snapshot!(format!("{:#?}", &spans), @r###"
+    [
+        Span {
+            start_line: 1,
+            start_col: 0,
+            start_offset: 0,
+            end_line: 1,
+            end_col: 65535,
+            end_offset: 70000,
+        },
+        Span {
+            start_line: 1,
+            start_col: 65535,
+            start_offset: 70000,
+            end_line: 1,
+            end_col: 65535,
+            end_offset: 70002,
+        },
+        Span {
+            start_line: 1,
+            start_col: 65535,
+            start_offset: 70003,
+            end_line: 1,
+            end_col: 65535,
+            end_offset: 70004,
+        },
+        Span {
+            start_line: 1,
+            start_col: 65535,
+            start_offset: 70005,
+            end_line: 1,
+            end_col: 65535,
+            end_offset: 70007,
+        },
+    ]
+    "###);
+}