Repository: mitsuhiko/minijinja (language: rust)

Issue:
Binary (and octal and hexadecimal) integer literals
For a specific compatibility use case, I would be interested in MiniJinja supporting Python's `0b10101` (and, for consistency, `0xabcdef` and `0o1234`) integer literals. The relevant code seems easy enough to hack:

https://github.com/mitsuhiko/minijinja/blob/68f6a2d5d05dc34e95e914f4c56c2608d623a366/minijinja/src/compiler/lexer.rs#L218-L263

I would be happy to work on a PR if there is upstream interest. Is there? :slightly_smiling_face: 


Discussion / hints:
Yeah. Happy to accept a PR on this.

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b92931f6..a41af88f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@ All notable changes to MiniJinja are documented here.
   should be performed.  #424
 - Added support for `Value::from_iterator`, `IteratorObject` and
   `ObjectKind::Iterator`.  #426
+- Added support for `0b`/`0o`/`0x` prefixed integer literals for
+  binary, octal and hexadecimal notation.  #433
 
 ## 1.0.12
 
diff --git a/minijinja/src/compiler/lexer.rs b/minijinja/src/compiler/lexer.rs
index ba6f5a62..b46aaf8e 100644
--- a/minijinja/src/compiler/lexer.rs
+++ b/minijinja/src/compiler/lexer.rs
@@ -218,6 +218,7 @@ impl<'s> TokenizerState<'s> {
     fn eat_number(&mut self) -> Result<(Token<'s>, Span), Error> {
         #[derive(Copy, Clone)]
         enum State {
+            RadixInteger, // 0x10
             Integer,      // 123
             Fraction,     // .123
             Exponent,     // E | e
@@ -225,7 +226,21 @@ impl<'s> TokenizerState<'s> {
         }
 
         let old_loc = self.loc();
-        let mut state = State::Integer;
+
+        let radix = match self.rest.as_bytes().get(..2) {
+            Some(b"0b" | b"0B") => 2,
+            Some(b"0o" | b"0O") => 8,
+            Some(b"0x" | b"0X") => 16,
+            _ => 10,
+        };
+
+        let mut state = if radix == 10 {
+            State::Integer
+        } else {
+            self.advance(2);
+            State::RadixInteger
+        };
+
         let mut num_len = self
             .rest
             .as_bytes()
@@ -239,11 +254,12 @@ impl<'s> TokenizerState<'s> {
                 (b'+' | b'-', State::Exponent) => State::ExponentSign,
                 (b'0'..=b'9', State::Exponent) => State::ExponentSign,
                 (b'0'..=b'9', state) => state,
+                (b'a'..=b'f' | b'A'..=b'F', State::RadixInteger) if radix == 16 => state,
                 _ => break,
             };
             num_len += 1;
         }
-        let is_float = !matches!(state, State::Integer);
+        let is_float = !matches!(state, State::Integer | State::RadixInteger);
 
         let num = self.advance(num_len);
         Ok((
@@ -251,10 +267,10 @@ impl<'s> TokenizerState<'s> {
                 num.parse()
                     .map(Token::Float)
                     .map_err(|_| self.syntax_error("invalid float"))
-            } else if let Ok(int) = num.parse() {
+            } else if let Ok(int) = u64::from_str_radix(num, radix) {
                 Ok(Token::Int(int))
             } else {
-                num.parse()
+                u128::from_str_radix(num, radix)
                     .map(Token::Int128)
                     .map_err(|_| self.syntax_error("invalid integer"))
             }),
diff --git a/minijinja/src/syntax.rs b/minijinja/src/syntax.rs
index 01dc51a7..47295e9f 100644
--- a/minijinja/src/syntax.rs
+++ b/minijinja/src/syntax.rs
@@ -80,7 +80,8 @@
 //! - `"Hello World"`: Everything between two double or single quotes is a string. They are
 //!   useful whenever you need a string in the template (e.g. as arguments to function calls
 //!   and filters, or just to extend or include a template).
-//! - `42`: Integers are whole numbers without a decimal part.
+//! - `42`: Integers are whole numbers without a decimal part.  They can be prefixed with
+//!   `0b` to indicate binary, `0o` to indicate octal and `0x` to indicate hexadecimal.
 //! - `42.0`: Floating point numbers can be written using a `.` as a decimal mark.
 //! - `['list', 'of', 'objects']`: Everything between two brackets is a list. Lists are useful
 //!   for storing sequential data to be iterated over.


Test patch:
diff --git a/minijinja/tests/lexer-inputs/literals.txt b/minijinja/tests/lexer-inputs/literals.txt
index 2f696bf8..13e65eea 100644
--- a/minijinja/tests/lexer-inputs/literals.txt
+++ b/minijinja/tests/lexer-inputs/literals.txt
@@ -9,3 +9,9 @@
 {{ -170141183460469231731687303715884105728 }}
 {{ 170141183460469231731687303715884105727 }}
 {{ 340282366920938463463374607431768211455 }}
+{{ 0b1111 }}
+{{ 0B1001 }}
+{{ 0o777 }}
+{{ 0O777 }}
+{{ 0xdeadbeef }}
+{{ 0XDEADBEEF }}
diff --git a/minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap b/minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap
index aebdf2cf..6ea2ec66 100644
--- a/minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap
+++ b/minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap
@@ -1,6 +1,6 @@
 ---
 source: minijinja/tests/test_lexer.rs
-description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 2 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}"
+description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 2 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}"
 input_file: minijinja/tests/lexer-inputs/literals.txt
 ---
 VariableStart
@@ -115,4 +115,51 @@ VariableEnd
   "}}"
 TemplateData("\n")
   "\n"
-
+VariableStart
+  "{{"
+Int(15)
+  "0b1111"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Int(9)
+  "0B1001"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Int(511)
+  "0o777"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Int(511)
+  "0O777"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Int(3735928559)
+  "0xdeadbeef"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Int(3735928559)
+  "0XDEADBEEF"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
diff --git a/minijinja/tests/test_value.rs b/minijinja/tests/test_value.rs
index 33a1ca75..88845050 100644
--- a/minijinja/tests/test_value.rs
+++ b/minijinja/tests/test_value.rs
@@ -4,7 +4,7 @@ use insta::assert_snapshot;
 use similar_asserts::assert_eq;
 
 use minijinja::value::{Kwargs, Object, ObjectKind, Rest, SeqObject, StructObject, Value};
-use minijinja::{args, context, render, Environment, Error};
+use minijinja::{args, render, Environment, Error};
 
 #[test]
 fn test_sort() {