Repository: mitsuhiko/minijinja
Pull request #450 (2024-03-27 10:28:50)

Add the tokenize function back
Turns out it's still useful.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 09d3a70..95e4bf4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,8 +7,9 @@ All notable changes to MiniJinja are documented here.
 - Tolerate underscores in number literals.  #443
 - Added support for `trim_blocks` and `lstrip_blocks` for Jinja2
   compatibility.  #447
-- Changed API of `unstable_machinery`.  The `tokenize` function is gone
-  and was replaced with the `Tokenizer` struct.  `parse_expr` was added,
+- Changed API of `unstable_machinery`.  The `tokenize` function got an
+  extra argument for the `WhitespaceConfig`.  It's however recommended
+  to use the new `Tokenizer` struct instead.  `parse_expr` was added,
   the `parse` function now takes a `SyntaxConfig` and `WhitespaceConfig`.  #447
 
 ## 1.0.15
diff --git a/minijinja-cli/src/main.rs b/minijinja-cli/src/main.rs
index 2afabdf..9c958a8 100644
--- a/minijinja-cli/src/main.rs
+++ b/minijinja-cli/src/main.rs
@@ -7,7 +7,9 @@ use std::{fs, io};
 
 use anyhow::{anyhow, bail, Context, Error};
 use clap::ArgMatches;
-use minijinja::machinery::{get_compiled_template, parse, Instructions, Tokenizer};
+use minijinja::machinery::{
+    get_compiled_template, parse, tokenize, Instructions, WhitespaceConfig,
+};
 use minijinja::{
     context, AutoEscape, Environment, Error as MError, ErrorKind, UndefinedBehavior, Value,
 };
@@ -386,10 +388,17 @@ fn execute() -> Result<i32, Error> {
             }
             "tokens" => {
                 let tmpl = env.get_template(&template)?;
-                let mut tokenizer =
-                    Tokenizer::new(tmpl.source(), false, Default::default(), Default::default());
-                let tokens: Result<Vec<_>, _> =
-                    std::iter::from_fn(move || tokenizer.next_token().transpose()).collect();
+                let tokens: Result<Vec<_>, _> = tokenize(
+                    tmpl.source(),
+                    false,
+                    Default::default(),
+                    WhitespaceConfig {
+                        lstrip_blocks: matches.get_flag("lstrip-blocks"),
+                        trim_blocks: matches.get_flag("trim-blocks"),
+                        ..Default::default()
+                    },
+                )
+                .collect();
                 for (token, _) in tokens? {
                     writeln!(&mut output, "{:?}", token)?;
                 }
diff --git a/minijinja/src/compiler/lexer.rs b/minijinja/src/compiler/lexer.rs
index cf629f6..63211ca 100644
--- a/minijinja/src/compiler/lexer.rs
+++ b/minijinja/src/compiler/lexer.rs
@@ -746,22 +746,26 @@ impl<'s> Tokenizer<'s> {
     }
 }
 
+/// Utility function to quickly tokenize into an iterator.
+#[allow(unused)]
+pub fn tokenize(
+    input: &str,
+    in_expr: bool,
+    syntax_config: SyntaxConfig,
+    whitespace_config: WhitespaceConfig,
+) -> impl Iterator<Item = Result<(Token<'_>, Span), Error>> {
+    // This function is unused in minijinja itself, it's only used in tests and in the
+    // unstable machinery as a convenient alternative to the tokenizer.
+    let mut tokenizer = Tokenizer::new(input, in_expr, syntax_config, whitespace_config);
+    std::iter::from_fn(move || tokenizer.next_token().transpose())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     use similar_asserts::assert_eq;
 
-    pub fn tokenize(
-        input: &str,
-        in_expr: bool,
-        syntax_config: SyntaxConfig,
-    ) -> impl Iterator<Item = Result<(Token<'_>, Span), Error>> {
-        let mut tokenizer =
-            Tokenizer::new(input, in_expr, syntax_config, WhitespaceConfig::default());
-        std::iter::from_fn(move || tokenizer.next_token().transpose())
-    }
-
     #[test]
     fn test_find_marker() {
         let syntax = SyntaxConfig::default();
@@ -838,14 +842,15 @@ mod tests {
     #[test]
     fn test_basic_identifiers() {
         fn assert_ident(s: &str) {
-            match tokenize(s, true, Default::default()).next() {
+            match tokenize(s, true, Default::default(), Default::default()).next() {
                 Some(Ok((Token::Ident(ident), _))) if ident == s => {}
                 _ => panic!("did not get a matching token result: {s:?}"),
             }
         }
 
         fn assert_not_ident(s: &str) {
-            let res = tokenize(s, true, Default::default()).collect::<Result<Vec<_>, _>>();
+            let res = tokenize(s, true, Default::default(), Default::default())
+                .collect::<Result<Vec<_>, _>>();
             if let Ok(tokens) = res {
                 if let &[(Token::Ident(_), _)] = &tokens[..] {
                     panic!("got a single ident for {s:?}")
diff --git a/minijinja/src/lib.rs b/minijinja/src/lib.rs
index 9c22cf5..0109111 100644
--- a/minijinja/src/lib.rs
+++ b/minijinja/src/lib.rs
@@ -260,7 +260,7 @@ pub mod machinery {
     pub use crate::compiler::ast;
     pub use crate::compiler::codegen::CodeGenerator;
     pub use crate::compiler::instructions::{Instruction, Instructions};
-    pub use crate::compiler::lexer::{SyntaxConfig, Tokenizer, WhitespaceConfig};
+    pub use crate::compiler::lexer::{tokenize, SyntaxConfig, Tokenizer, WhitespaceConfig};
     pub use crate::compiler::parser::{parse, parse_expr};
     pub use crate::compiler::tokens::{Span, Token};
     pub use crate::template::{CompiledTemplate, TemplateConfig};


Test patch:
diff --git a/minijinja/tests/test_lexer.rs b/minijinja/tests/test_lexer.rs
index 315ab89..290b948 100644
--- a/minijinja/tests/test_lexer.rs
+++ b/minijinja/tests/test_lexer.rs
@@ -1,8 +1,8 @@
 #![cfg(feature = "unstable_machinery")]
 use minijinja::machinery::{
-    make_syntax_config, Span, SyntaxConfig, Token, Tokenizer, WhitespaceConfig,
+    make_syntax_config, tokenize, Span, SyntaxConfig, Token, WhitespaceConfig,
 };
-use minijinja::{Error, Syntax};
+use minijinja::Syntax;
 
 use std::fmt::Write;
 
@@ -42,15 +42,6 @@ impl TestSettings {
     }
 }
 
-pub fn tokenize(
-    input: &str,
-    syntax_config: SyntaxConfig,
-    whitespace_config: WhitespaceConfig,
-) -> Result<Vec<(Token<'_>, Span)>, Error> {
-    let mut tokenizer = Tokenizer::new(input, false, syntax_config, whitespace_config);
-    std::iter::from_fn(move || tokenizer.next_token().transpose()).collect()
-}
-
 fn stringify_tokens(tokens: Vec<(Token<'_>, Span)>, contents: &str) -> String {
     let mut stringified = String::new();
     for (token, span) in tokens {
@@ -71,7 +62,8 @@ fn test_lexer() {
         let settings: TestSettings = serde_json::from_str(iter.next().unwrap()).unwrap();
         let (syntax_config, whitespace_config) = settings.into_configs();
         let contents = iter.next().unwrap();
-        let tokens = tokenize(contents, syntax_config, whitespace_config);
+        let tokens: Result<Vec<_>, _> =
+            tokenize(contents, false, syntax_config, whitespace_config).collect();
         insta::with_settings!({
             description => contents.trim_end(),
             omit_expression => true
@@ -85,14 +77,16 @@ fn test_lexer() {
 #[test]
 fn test_trim_blocks() {
     let input = "{% block foo %}\nbar{% endblock %}";
-    let tokens = tokenize(
+    let tokens: Result<Vec<_>, _> = tokenize(
         input,
+        false,
         Default::default(),
         WhitespaceConfig {
             trim_blocks: true,
             ..Default::default()
         },
-    );
+    )
+    .collect();
     let stringified = stringify_tokens(tokens.unwrap(), input);
     insta::assert_snapshot!(&stringified, @r###"
     BlockStart