Repository: reubeno/brush
Pull request #550 (2025-06-09 03:52:33)

test(tokenizer): adopt insta for tokenizer tests
Follows the pattern of #545 -- starts using `insta` for tokenizer tests. This brings stronger strictness, since full source position information will now be captured too.

github-actions[bot]: ## Test Results
    3 files     17 suites   3m 38s ⏱️
  771 tests   771 ✅ 0 💤 0 ❌
2 299 runs  2 299 ✅ 0 💤 0 ❌

Results for commit 6a479ce9.

[test-results]:data:application/gzip;base64,H4sIAEhdRmgC/12PwQrDIBAFfyV47iGrUrU/U8yqIE1iMXoq/fduJGmTHmceDLwXC3H0C7t14tKxpcbSABSRq9mWmGZiDpoEbWVdlYKd7ktF/FeP+CTVf0WwcTwJn3PKm8l1XpucG7PR3jypX7PxIdn4WMQ0TbEQsKuVyqA3gwPthaQXToPupRLOBi0AUAygULL3B0tubCYIAQAA

github-actions[bot]: # Performance Benchmark Report
| Benchmark name                         | Baseline (μs) |  Test/PR (μs) |    Delta (μs) | Delta %         |
| -------------------------------------- | ------------- | ------------- | ------------- | --------------- |
| `clone_shell_object                  ` | `   18.13 μs` | `   18.52 μs` | `    0.39 μs` | `🟠 +2.18%    ` |
| `eval_arithmetic                     ` | `    0.17 μs` | `    0.17 μs` | `    0.00 μs` | `⚪  Unchanged` |
| `expand_one_string                   ` | `    1.88 μs` | `    1.90 μs` | `    0.02 μs` | `⚪  Unchanged` |
| `for_loop                            ` | `   23.37 μs` | `   22.83 μs` | `   -0.54 μs` | `🟢 -2.29%    ` |
| `function_call                       ` | `    2.60 μs` | `    2.53 μs` | `   -0.07 μs` | `⚪  Unchanged` |
| `instantiate_shell                   ` | `   57.73 μs` | `   58.27 μs` | `    0.54 μs` | `⚪  Unchanged` |
| `instantiate_shell_with_init_scripts ` | `25231.34 μs` | `25186.80 μs` | `  -44.54 μs` | `⚪  Unchanged` |
| `parse_bash_completion               ` | ` 1668.67 μs` | ` 1702.32 μs` | `   33.66 μs` | `🟠 +2.02%    ` |
| `parse_sample_script                 ` | `    1.74 μs` | `    1.80 μs` | `    0.06 μs` | `⚪  Unchanged` |
| `run_echo_builtin_command            ` | `   15.42 μs` | `   15.18 μs` | `   -0.24 μs` | `⚪  Unchanged` |
| `run_one_external_command            ` | ` 2381.55 μs` | ` 2328.21 μs` | `  -53.34 μs` | `⚪  Unchanged` |
| `tokenize_sample_script              ` | `    2.87 μs` | `    3.00 μs` | `    0.13 μs` | `🟠 +4.39%    ` |

# Code Coverage Report: Only Changed Files listed

| Package | Base Coverage | New Coverage | Difference |
| ------- | ------------- | ------------ | ---------- |
| brush-parser/src/tokenizer.rs | 🟢 95.92% | 🟢 94.96% | 🔴 -0.96% |
| **Overall Coverage** | **🟢 74.56%** | **🟢 74.25%** | **🔴 -0.31%** |

_Minimum allowed coverage is_ `70%`_, this run produced_ `74.25%`

# Test Summary: bash-completion test suite
| Outcome            | Count                   | Percentage |
| ------------------ | ----------------------: | ---------: |
| ✅ Pass            | 1323            | <span style='color:green'>62.73</span> |
| ❗️ Error           | 230           | <span style='color:red'>10.91</span> |
| ❌ Fail            | 178            | <span style='color:red'>8.44</span> |
| ⏩ Skip            | 363            | 17.21 |
| ❎ Expected Fail   | 14   | 0.66 |
| ✔️ Unexpected Pass | 1 | <span style='color:red'>0.05</span> |
| 📊 Total           | 2109           | 100.00 |

<!-- Sticky Pull Request Comment -->

Patch:
diff --git a/brush-parser/src/tokenizer.rs b/brush-parser/src/tokenizer.rs
index 0c14b0c..29b6ae3 100644
--- a/brush-parser/src/tokenizer.rs
+++ b/brush-parser/src/tokenizer.rs
@@ -31,12 +31,15 @@ pub(crate) enum TokenEndReason {
 #[derive(Clone, Default, Debug)]
 #[cfg_attr(feature = "fuzz-testing", derive(arbitrary::Arbitrary))]
 #[cfg_attr(test, derive(PartialEq, Eq, serde::Serialize))]
+#[cfg_attr(test, serde(rename = "Pos"))]
 pub struct SourcePosition {
     /// The 0-based index of the character in the input stream.
+    #[cfg_attr(test, serde(rename = "idx"))]
     pub index: i32,
     /// The 1-based line number.
     pub line: i32,
     /// The 1-based column number.
+    #[cfg_attr(test, serde(rename = "col"))]
     pub column: i32,
 }
 
@@ -50,6 +53,7 @@ impl Display for SourcePosition {
 #[derive(Clone, Default, Debug)]
 #[cfg_attr(feature = "fuzz-testing", derive(arbitrary::Arbitrary))]
 #[cfg_attr(test, derive(PartialEq, Eq, serde::Serialize))]
+#[cfg_attr(test, serde(rename = "Loc"))]
 pub struct TokenLocation {
     /// The start position of the token.
     pub start: SourcePosition,
@@ -63,8 +67,10 @@ pub struct TokenLocation {
 #[cfg_attr(test, derive(PartialEq, Eq, serde::Serialize))]
 pub enum Token {
     /// An operator token.
+    #[cfg_attr(test, serde(rename = "Op"))]
     Operator(String, TokenLocation),
     /// A word token.
+    #[cfg_attr(test, serde(rename = "W"))]
     Word(String, TokenLocation),
 }
 
@@ -1275,8 +1281,22 @@ mod tests {
 
     use super::*;
     use anyhow::Result;
+    use insta::assert_ron_snapshot;
     use pretty_assertions::{assert_eq, assert_matches};
 
+    #[derive(serde::Serialize)]
+    struct TokenizerResult<'a> {
+        input: &'a str,
+        result: Vec<Token>,
+    }
+
+    fn test_tokenizer(input: &str) -> Result<TokenizerResult<'_>> {
+        Ok(TokenizerResult {
+            input,
+            result: tokenize_str(input)?,
+        })
+    }
+
     #[test]
     fn tokenize_empty() -> Result<()> {
         let tokens = tokenize_str("")?;
@@ -1286,212 +1306,91 @@ mod tests {
 
     #[test]
     fn tokenize_line_continuation() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"a\
-bc",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "abc"
-        );
+bc"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_operators() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("a>>b")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Operator(..), t3 @ Token::Word(..)] if
-                t1.to_str() == "a" &&
-                t2.to_str() == ">>" &&
-                t3.to_str() == "b"
-        );
+        assert_ron_snapshot!(test_tokenizer("a>>b")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_comment() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"a #comment
-",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..), t2 @ Token::Operator(..)] if
-                t1.to_str() == "a" &&
-                t2.to_str() == "\n"
-        );
+"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_comment_at_eof() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r"a #comment")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "a"
-        );
+        assert_ron_snapshot!(test_tokenizer(r"a #comment")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_empty_here_doc() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"cat <<HERE
 HERE
-",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Operator(..),
-             t3 @ Token::Word(..),
-             t4 @ Token::Word(..),
-             t5 @ Token::Word(..),
-             t6 @ Token::Operator(..)] if
-                t1.to_str() == "cat" &&
-                t2.to_str() == "<<" &&
-                t3.to_str() == "HERE" &&
-                t4.to_str() == "" &&
-                t5.to_str() == "HERE" &&
-                t6.to_str() == "\n"
-        );
+"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_here_doc() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"cat <<HERE
 SOMETHING
 HERE
 echo after
-",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Operator(..),
-             t3 @ Token::Word(..),
-             t4 @ Token::Word(..),
-             t5 @ Token::Word(..),
-             t6 @ Token::Operator(..),
-             t7 @ Token::Word(..),
-             t8 @ Token::Word(..),
-             t9 @ Token::Operator(..)] if
-                t1.to_str() == "cat" &&
-                t2.to_str() == "<<" &&
-                t3.to_str() == "HERE" &&
-                t4.to_str() == "SOMETHING\n" &&
-                t5.to_str() == "HERE" &&
-                t6.to_str() == "\n" &&
-                t7.to_str() == "echo" &&
-                t8.to_str() == "after" &&
-                t9.to_str() == "\n"
-        );
+"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_here_doc_with_tab_removal() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"cat <<-HERE
 	SOMETHING
 	HERE
-",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Operator(..),
-             t3 @ Token::Word(..),
-             t4 @ Token::Word(..),
-             t5 @ Token::Word(..),
-             t6 @ Token::Operator(..)] if
-                t1.to_str() == "cat" &&
-                t2.to_str() == "<<-" &&
-                t3.to_str() == "HERE" &&
-                t4.to_str() == "SOMETHING\n" &&
-                t5.to_str() == "HERE" &&
-                t6.to_str() == "\n"
-        );
+"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_here_doc_with_other_tokens() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"cat <<EOF | wc -l
 A B C
 1 2 3
 D E F
 EOF
-",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Operator(..),
-             t3 @ Token::Word(..),
-             t4 @ Token::Word(..),
-             t5 @ Token::Word(..),
-             t6 @ Token::Operator(..),
-             t7 @ Token::Word(..),
-             t8 @ Token::Word(..),
-             t9 @ Token::Operator(..)] if
-                t1.to_str() == "cat" &&
-                t2.to_str() == "<<" &&
-                t3.to_str() == "EOF" &&
-                t4.to_str() == "A B C\n1 2 3\nD E F\n" &&
-                t5.to_str() == "EOF" &&
-                t6.to_str() == "|" &&
-                t7.to_str() == "wc" &&
-                t8.to_str() == "-l" &&
-                t9.to_str() == "\n"
-        );
-
+"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_multiple_here_docs() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"cat <<HERE1 <<HERE2
 SOMETHING
 HERE1
 OTHER
 HERE2
 echo after
-",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Operator(..),
-             t3 @ Token::Word(..),
-             t4 @ Token::Word(..),
-             t5 @ Token::Word(..),
-             t6 @ Token::Operator(..),
-             t7 @ Token::Word(..),
-             t8 @ Token::Word(..),
-             t9 @ Token::Word(..),
-             t10 @ Token::Operator(..),
-             t11 @ Token::Word(..),
-             t12 @ Token::Word(..),
-             t13 @ Token::Operator(..)] if
-                t1.to_str() == "cat" &&
-                t2.to_str() == "<<" &&
-                t3.to_str() == "HERE1" &&
-                t4.to_str() == "SOMETHING\n" &&
-                t5.to_str() == "HERE1" &&
-                t6.to_str() == "<<" &&
-                t7.to_str() == "HERE2" &&
-                t8.to_str() == "OTHER\n" &&
-                t9.to_str() == "HERE2" &&
-                t10.to_str() == "\n" &&
-                t11.to_str() == "echo" &&
-                t12.to_str() == "after" &&
-                t13.to_str() == "\n"
-        );
+"
+        )?);
         Ok(())
     }
 
@@ -1516,61 +1415,37 @@ SOMETHING
 
     #[test]
     fn tokenize_here_doc_in_command_substitution() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"echo $(cat <<HERE
 TEXT
 HERE
-)",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Word(..)] if
-                t1.to_str() == "echo" &&
-                t2.to_str() == "$(cat <<HERE\nTEXT\nHERE\n)"
-        );
+)"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_complex_here_docs_in_command_substitution() -> Result<()> {
-        let tokens = tokenize_str(
+        assert_ron_snapshot!(test_tokenizer(
             r"echo $(cat <<HERE1 <<HERE2 | wc -l
 TEXT
 HERE1
 OTHER
 HERE2
-)",
-        )?;
-        assert_matches!(
-            &tokens[..],
-            [t1 @ Token::Word(..),
-             t2 @ Token::Word(..)] if
-                t1.to_str() == "echo" &&
-                t2.to_str() == "$(cat <<HERE1 <<HERE2 | wc -l\nTEXT\nHERE1\nOTHER\nHERE2\n)"
-        );
+)"
+        )?);
         Ok(())
     }
 
     #[test]
     fn tokenize_simple_backquote() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r"echo `echo hi`")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..)] if
-                t1.to_str() == "echo" &&
-                t2.to_str() == "`echo hi`"
-        );
+        assert_ron_snapshot!(test_tokenizer(r"echo `echo hi`")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_backquote_with_escape() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r"echo `echo\`hi`")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..)] if
-                t1.to_str() == "echo" &&
-                t2.to_str() == r"`echo\`hi`"
-        );
+        assert_ron_snapshot!(test_tokenizer(r"echo `echo\`hi`")?);
         Ok(())
     }
 
@@ -1592,43 +1467,25 @@ HERE2
 
     #[test]
     fn tokenize_command_substitution() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("a$(echo hi)b c")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..)] if
-                t1.to_str() == "a$(echo hi)b" &&
-                t2.to_str() == "c"
-        );
+        assert_ron_snapshot!(test_tokenizer("a$(echo hi)b c")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_command_substitution_with_subshell() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("$( (:) )")?[..],
-            [t @ Token::Word(..)] if t.to_str() == "$( (:) )"
-        );
+        assert_ron_snapshot!(test_tokenizer("$( (:) )")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_command_substitution_containing_extglob() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("echo $(echo !(x))")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..)] if
-                t1.to_str() == "echo" &&
-                t2.to_str() == "$(echo !(x))"
-        );
+        assert_ron_snapshot!(test_tokenizer("echo $(echo !(x))")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_arithmetic_expression() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("a$((1+2))b c")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..)] if
-                t1.to_str() == "a$((1+2))b" &&
-                t2.to_str() == "c"
-        );
+        assert_ron_snapshot!(test_tokenizer("a$((1+2))b c")?);
         Ok(())
     }
 
@@ -1636,58 +1493,29 @@ HERE2
     fn tokenize_arithmetic_expression_with_space() -> Result<()> {
         // N.B. The spacing comes out a bit odd, but it gets processed okay
         // by later stages.
-        assert_matches!(
-            &tokenize_str("$(( 1 ))")?[..],
-            [t1 @ Token::Word(..)] if
-                t1.to_str() == "$(( 1 ))"
-        );
+        assert_ron_snapshot!(test_tokenizer("$(( 1 ))")?);
         Ok(())
     }
     #[test]
     fn tokenize_arithmetic_expression_with_parens() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("$(( (0) ))")?[..],
-            [t1 @ Token::Word(..)] if
-                t1.to_str() == "$(( (0) ))"
-        );
+        assert_ron_snapshot!(test_tokenizer("$(( (0) ))")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_special_parameters() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("$$")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "$$"
-        );
-        assert_matches!(
-            &tokenize_str("$@")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "$@"
-        );
-        assert_matches!(
-            &tokenize_str("$!")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "$!"
-        );
-        assert_matches!(
-            &tokenize_str("$?")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "$?"
-        );
-        assert_matches!(
-            &tokenize_str("$*")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "$*"
-        );
+        assert_ron_snapshot!(test_tokenizer("$$")?);
+        assert_ron_snapshot!(test_tokenizer("$@")?);
+        assert_ron_snapshot!(test_tokenizer("$!")?);
+        assert_ron_snapshot!(test_tokenizer("$?")?);
+        assert_ron_snapshot!(test_tokenizer("$*")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_unbraced_parameter_expansion() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("$x")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "$x"
-        );
-        assert_matches!(
-            &tokenize_str("a$x")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "a$x"
-        );
+        assert_ron_snapshot!(test_tokenizer("$x")?);
+        assert_ron_snapshot!(test_tokenizer("a$x")?);
         Ok(())
     }
 
@@ -1701,86 +1529,50 @@ HERE2
 
     #[test]
     fn tokenize_braced_parameter_expansion() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("${x}")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "${x}"
-        );
-        assert_matches!(
-            &tokenize_str("a${x}b")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == "a${x}b"
-        );
+        assert_ron_snapshot!(test_tokenizer("${x}")?);
+        assert_ron_snapshot!(test_tokenizer("a${x}b")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_braced_parameter_expansion_with_escaping() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r"a${x\}}b")?[..],
-            [t1 @ Token::Word(..)] if t1.to_str() == r"a${x\}}b"
-        );
+        assert_ron_snapshot!(test_tokenizer(r"a${x\}}b")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_whitespace() -> Result<()> {
-        assert_matches!(
-            &tokenize_str("1 2 3")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..), t3 @ Token::Word(..)] if
-                t1.to_str() == "1" &&
-                t2.to_str() == "2" &&
-                t3.to_str() == "3"
-        );
+        assert_ron_snapshot!(test_tokenizer("1 2 3")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_escaped_whitespace() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r"1\ 2 3")?[..],
-            [t1 @ Token::Word(..), t2 @ Token::Word(..)] if
-                t1.to_str() == r"1\ 2" &&
-                t2.to_str() == "3"
-        );
+        assert_ron_snapshot!(test_tokenizer(r"1\ 2 3")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_single_quote() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r"x'a b'y")?[..],
-            [t1 @ Token::Word(..)] if
-                t1.to_str() == r"x'a b'y"
-        );
+        assert_ron_snapshot!(test_tokenizer(r"x'a b'y")?);
         Ok(())
     }
 
     #[test]
     fn tokenize_double_quote() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r#"x"a b"y"#)?[..],
-            [t1 @ Token::Word(..)] if
-                t1.to_str() == r#"x"a b"y"#
-        );
+        assert_ron_snapshot!(test_tokenizer(r#"x"a b"y"#)?);
         Ok(())
     }
 
     #[test]
     fn tokenize_double_quoted_command_substitution() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r#"x"$(echo hi)"y"#)?[..],
-            [t1 @ Token::Word(..)] if
-                t1.to_str() == r#"x"$(echo hi)"y"#
-        );
+        assert_ron_snapshot!(test_tokenizer(r#"x"$(echo hi)"y"#)?);
         Ok(())
     }
 
     #[test]
     fn tokenize_double_quoted_arithmetic_expression() -> Result<()> {
-        assert_matches!(
-            &tokenize_str(r#"x"$((1+2))"y"#)?[..],
-            [t1 @ Token::Word(..)] if
-                t1.to_str() == r#"x"$((1+2))"y"#
-        );
+        assert_ron_snapshot!(test_tokenizer(r#"x"$((1+2))"y"#)?);
         Ok(())
     }
 


Test patch:
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap
new file mode 100644
index 0000000..6c5590f
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"a$((1+2))b c\")?"
+---
+TokenizerResult(
+  input: "a$((1+2))b c",
+  result: [
+    W("a$((1+2))b", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 10,
+        line: 1,
+        col: 11,
+      ),
+    )),
+    W("c", Loc(
+      start: Pos(
+        idx: 11,
+        line: 1,
+        col: 12,
+      ),
+      end: Pos(
+        idx: 12,
+        line: 1,
+        col: 13,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap
new file mode 100644
index 0000000..6d5f060
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_parens.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$(( (0) ))\")?"
+---
+TokenizerResult(
+  input: "$(( (0) ))",
+  result: [
+    W("$(( (0) ))", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 10,
+        line: 1,
+        col: 11,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap
new file mode 100644
index 0000000..bd22d87
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_arithmetic_expression_with_space.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$(( 1 ))\")?"
+---
+TokenizerResult(
+  input: "$(( 1 ))",
+  result: [
+    W("$(( 1 ))", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 8,
+        line: 1,
+        col: 9,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap
new file mode 100644
index 0000000..6f11f8c
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_backquote_with_escape.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"echo `echo\\`hi`\")?"
+---
+TokenizerResult(
+  input: "echo `echo\\`hi`",
+  result: [
+    W("echo", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+    W("`echo\\`hi`", Loc(
+      start: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 15,
+        line: 1,
+        col: 16,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap
new file mode 100644
index 0000000..ee16adb
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion-2.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"a${x}b\")?"
+---
+TokenizerResult(
+  input: "a${x}b",
+  result: [
+    W("a${x}b", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap
new file mode 100644
index 0000000..86daffc
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"${x}\")?"
+---
+TokenizerResult(
+  input: "${x}",
+  result: [
+    W("${x}", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap
new file mode 100644
index 0000000..3a03b06
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_braced_parameter_expansion_with_escaping.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"a${x\\}}b\")?"
+---
+TokenizerResult(
+  input: "a${x\\}}b",
+  result: [
+    W("a${x\\}}b", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 8,
+        line: 1,
+        col: 9,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap
new file mode 100644
index 0000000..d826ffb
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"a$(echo hi)b c\")?"
+---
+TokenizerResult(
+  input: "a$(echo hi)b c",
+  result: [
+    W("a$(echo hi)b", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 12,
+        line: 1,
+        col: 13,
+      ),
+    )),
+    W("c", Loc(
+      start: Pos(
+        idx: 13,
+        line: 1,
+        col: 14,
+      ),
+      end: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap
new file mode 100644
index 0000000..fcc6e14
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_containing_extglob.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"echo $(echo !(x))\")?"
+---
+TokenizerResult(
+  input: "echo $(echo !(x))",
+  result: [
+    W("echo", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+    W("$(echo !(x))", Loc(
+      start: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 17,
+        line: 1,
+        col: 18,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap
new file mode 100644
index 0000000..8f26c72
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_command_substitution_with_subshell.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$( (:) )\")?"
+---
+TokenizerResult(
+  input: "$( (:) )",
+  result: [
+    W("$( (:) )", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 8,
+        line: 1,
+        col: 9,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap
new file mode 100644
index 0000000..509a967
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"a #comment\n\")?"
+---
+TokenizerResult(
+  input: "a #comment\n",
+  result: [
+    W("a", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 1,
+        line: 1,
+        col: 2,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+      end: Pos(
+        idx: 11,
+        line: 2,
+        col: 1,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap
new file mode 100644
index 0000000..77a2501
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_comment_at_eof.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"a #comment\")?"
+---
+TokenizerResult(
+  input: "a #comment",
+  result: [
+    W("a", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 1,
+        line: 1,
+        col: 2,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap
new file mode 100644
index 0000000..09c3871
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_complex_here_docs_in_command_substitution.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"echo $(cat <<HERE1 <<HERE2 | wc -l\nTEXT\nHERE1\nOTHER\nHERE2\n)\")?"
+---
+TokenizerResult(
+  input: "echo $(cat <<HERE1 <<HERE2 | wc -l\nTEXT\nHERE1\nOTHER\nHERE2\n)",
+  result: [
+    W("echo", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+    W("$(cat <<HERE1 <<HERE2 | wc -l\nTEXT\nHERE1\nOTHER\nHERE2\n)", Loc(
+      start: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 59,
+        line: 6,
+        col: 2,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quote.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quote.snap
new file mode 100644
index 0000000..15e7326
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quote.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r#\"x\"a b\"y\"#)?"
+---
+TokenizerResult(
+  input: "x\"a b\"y",
+  result: [
+    W("x\"a b\"y", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 7,
+        line: 1,
+        col: 8,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quoted_arithmetic_expression.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quoted_arithmetic_expression.snap
new file mode 100644
index 0000000..f73e401
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quoted_arithmetic_expression.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r#\"x\"$((1+2))\"y\"#)?"
+---
+TokenizerResult(
+  input: "x\"$((1+2))\"y",
+  result: [
+    W("x\"$((1+2))\"y", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 12,
+        line: 1,
+        col: 13,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quoted_command_substitution.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quoted_command_substitution.snap
new file mode 100644
index 0000000..54f18a9
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_double_quoted_command_substitution.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r#\"x\"$(echo hi)\"y\"#)?"
+---
+TokenizerResult(
+  input: "x\"$(echo hi)\"y",
+  result: [
+    W("x\"$(echo hi)\"y", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_empty_here_doc.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_empty_here_doc.snap
new file mode 100644
index 0000000..78d04d3
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_empty_here_doc.snap
@@ -0,0 +1,81 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"cat <<HERE\nHERE\n\")?"
+---
+TokenizerResult(
+  input: "cat <<HERE\nHERE\n",
+  result: [
+    W("cat", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    Op("<<", Loc(
+      start: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+      end: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+    )),
+    W("HERE", Loc(
+      start: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+      end: Pos(
+        idx: 10,
+        line: 1,
+        col: 11,
+      ),
+    )),
+    W("", Loc(
+      start: Pos(
+        idx: 11,
+        line: 2,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 16,
+        line: 3,
+        col: 1,
+      ),
+    )),
+    W("HERE", Loc(
+      start: Pos(
+        idx: 16,
+        line: 3,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 16,
+        line: 3,
+        col: 1,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 10,
+        line: 1,
+        col: 11,
+      ),
+      end: Pos(
+        idx: 11,
+        line: 2,
+        col: 1,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_escaped_whitespace.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_escaped_whitespace.snap
new file mode 100644
index 0000000..70cc57e
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_escaped_whitespace.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"1\\ 2 3\")?"
+---
+TokenizerResult(
+  input: "1\\ 2 3",
+  result: [
+    W("1\\ 2", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+    W("3", Loc(
+      start: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc.snap
new file mode 100644
index 0000000..21af5aa
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc.snap
@@ -0,0 +1,117 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"cat <<HERE\nSOMETHING\nHERE\necho after\n\")?"
+---
+TokenizerResult(
+  input: "cat <<HERE\nSOMETHING\nHERE\necho after\n",
+  result: [
+    W("cat", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    Op("<<", Loc(
+      start: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+      end: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+    )),
+    W("HERE", Loc(
+      start: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+      end: Pos(
+        idx: 10,
+        line: 1,
+        col: 11,
+      ),
+    )),
+    W("SOMETHING\n", Loc(
+      start: Pos(
+        idx: 11,
+        line: 2,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 26,
+        line: 4,
+        col: 1,
+      ),
+    )),
+    W("HERE", Loc(
+      start: Pos(
+        idx: 26,
+        line: 4,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 26,
+        line: 4,
+        col: 1,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 10,
+        line: 1,
+        col: 11,
+      ),
+      end: Pos(
+        idx: 11,
+        line: 2,
+        col: 1,
+      ),
+    )),
+    W("echo", Loc(
+      start: Pos(
+        idx: 26,
+        line: 4,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 30,
+        line: 4,
+        col: 5,
+      ),
+    )),
+    W("after", Loc(
+      start: Pos(
+        idx: 31,
+        line: 4,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 36,
+        line: 4,
+        col: 11,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 36,
+        line: 4,
+        col: 11,
+      ),
+      end: Pos(
+        idx: 37,
+        line: 5,
+        col: 1,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_in_command_substitution.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_in_command_substitution.snap
new file mode 100644
index 0000000..0c569e5
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_in_command_substitution.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"echo $(cat <<HERE\nTEXT\nHERE\n)\")?"
+---
+TokenizerResult(
+  input: "echo $(cat <<HERE\nTEXT\nHERE\n)",
+  result: [
+    W("echo", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+    W("$(cat <<HERE\nTEXT\nHERE\n)", Loc(
+      start: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 29,
+        line: 4,
+        col: 2,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_with_other_tokens.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_with_other_tokens.snap
new file mode 100644
index 0000000..da18b41
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_with_other_tokens.snap
@@ -0,0 +1,117 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"cat <<EOF | wc -l\nA B C\n1 2 3\nD E F\nEOF\n\")?"
+---
+TokenizerResult(
+  input: "cat <<EOF | wc -l\nA B C\n1 2 3\nD E F\nEOF\n",
+  result: [
+    W("cat", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    Op("<<", Loc(
+      start: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+      end: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+    )),
+    W("EOF", Loc(
+      start: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+      end: Pos(
+        idx: 9,
+        line: 1,
+        col: 10,
+      ),
+    )),
+    W("A B C\n1 2 3\nD E F\n", Loc(
+      start: Pos(
+        idx: 18,
+        line: 2,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 40,
+        line: 6,
+        col: 1,
+      ),
+    )),
+    W("EOF", Loc(
+      start: Pos(
+        idx: 40,
+        line: 6,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 40,
+        line: 6,
+        col: 1,
+      ),
+    )),
+    Op("|", Loc(
+      start: Pos(
+        idx: 9,
+        line: 1,
+        col: 10,
+      ),
+      end: Pos(
+        idx: 11,
+        line: 1,
+        col: 12,
+      ),
+    )),
+    W("wc", Loc(
+      start: Pos(
+        idx: 12,
+        line: 1,
+        col: 13,
+      ),
+      end: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+    )),
+    W("-l", Loc(
+      start: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+      end: Pos(
+        idx: 17,
+        line: 1,
+        col: 18,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 17,
+        line: 1,
+        col: 18,
+      ),
+      end: Pos(
+        idx: 18,
+        line: 2,
+        col: 1,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_with_tab_removal.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_with_tab_removal.snap
new file mode 100644
index 0000000..e6ffefe
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_here_doc_with_tab_removal.snap
@@ -0,0 +1,81 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"cat <<-HERE\n\tSOMETHING\n\tHERE\n\")?"
+---
+TokenizerResult(
+  input: "cat <<-HERE\n\tSOMETHING\n\tHERE\n",
+  result: [
+    W("cat", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    Op("<<-", Loc(
+      start: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+      end: Pos(
+        idx: 7,
+        line: 1,
+        col: 8,
+      ),
+    )),
+    W("HERE", Loc(
+      start: Pos(
+        idx: 7,
+        line: 1,
+        col: 8,
+      ),
+      end: Pos(
+        idx: 11,
+        line: 1,
+        col: 12,
+      ),
+    )),
+    W("SOMETHING\n", Loc(
+      start: Pos(
+        idx: 12,
+        line: 2,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 29,
+        line: 4,
+        col: 1,
+      ),
+    )),
+    W("HERE", Loc(
+      start: Pos(
+        idx: 29,
+        line: 4,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 29,
+        line: 4,
+        col: 1,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 11,
+        line: 1,
+        col: 12,
+      ),
+      end: Pos(
+        idx: 12,
+        line: 2,
+        col: 1,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_line_continuation.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_line_continuation.snap
new file mode 100644
index 0000000..b1fe1de
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_line_continuation.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"a\\\nbc\")?"
+---
+TokenizerResult(
+  input: "a\\\nbc",
+  result: [
+    W("abc", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 5,
+        line: 2,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_multiple_here_docs.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_multiple_here_docs.snap
new file mode 100644
index 0000000..f204099
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_multiple_here_docs.snap
@@ -0,0 +1,165 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"cat <<HERE1 <<HERE2\nSOMETHING\nHERE1\nOTHER\nHERE2\necho after\n\")?"
+---
+TokenizerResult(
+  input: "cat <<HERE1 <<HERE2\nSOMETHING\nHERE1\nOTHER\nHERE2\necho after\n",
+  result: [
+    W("cat", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    Op("<<", Loc(
+      start: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+      end: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+    )),
+    W("HERE1", Loc(
+      start: Pos(
+        idx: 6,
+        line: 1,
+        col: 7,
+      ),
+      end: Pos(
+        idx: 11,
+        line: 1,
+        col: 12,
+      ),
+    )),
+    W("SOMETHING\n", Loc(
+      start: Pos(
+        idx: 20,
+        line: 2,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 36,
+        line: 4,
+        col: 1,
+      ),
+    )),
+    W("HERE1", Loc(
+      start: Pos(
+        idx: 36,
+        line: 4,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 36,
+        line: 4,
+        col: 1,
+      ),
+    )),
+    Op("<<", Loc(
+      start: Pos(
+        idx: 11,
+        line: 1,
+        col: 12,
+      ),
+      end: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+    )),
+    W("HERE2", Loc(
+      start: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+      end: Pos(
+        idx: 19,
+        line: 1,
+        col: 20,
+      ),
+    )),
+    W("OTHER\n", Loc(
+      start: Pos(
+        idx: 36,
+        line: 4,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 48,
+        line: 6,
+        col: 1,
+      ),
+    )),
+    W("HERE2", Loc(
+      start: Pos(
+        idx: 48,
+        line: 6,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 48,
+        line: 6,
+        col: 1,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 19,
+        line: 1,
+        col: 20,
+      ),
+      end: Pos(
+        idx: 20,
+        line: 2,
+        col: 1,
+      ),
+    )),
+    W("echo", Loc(
+      start: Pos(
+        idx: 48,
+        line: 6,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 52,
+        line: 6,
+        col: 5,
+      ),
+    )),
+    W("after", Loc(
+      start: Pos(
+        idx: 53,
+        line: 6,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 58,
+        line: 6,
+        col: 11,
+      ),
+    )),
+    Op("\n", Loc(
+      start: Pos(
+        idx: 58,
+        line: 6,
+        col: 11,
+      ),
+      end: Pos(
+        idx: 59,
+        line: 7,
+        col: 1,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_operators.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_operators.snap
new file mode 100644
index 0000000..9f607fe
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_operators.snap
@@ -0,0 +1,45 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"a>>b\")?"
+---
+TokenizerResult(
+  input: "a>>b",
+  result: [
+    W("a", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 1,
+        line: 1,
+        col: 2,
+      ),
+    )),
+    Op(">>", Loc(
+      start: Pos(
+        idx: 1,
+        line: 1,
+        col: 2,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    W("b", Loc(
+      start: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap
new file mode 100644
index 0000000..63b3d68
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_simple_backquote.snap
@@ -0,0 +1,33 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"echo `echo hi`\")?"
+---
+TokenizerResult(
+  input: "echo `echo hi`",
+  result: [
+    W("echo", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+    )),
+    W("`echo hi`", Loc(
+      start: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+      end: Pos(
+        idx: 14,
+        line: 1,
+        col: 15,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap
new file mode 100644
index 0000000..7fb9554
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_single_quote.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(r\"x'a b'y\")?"
+---
+TokenizerResult(
+  input: "x\'a b\'y",
+  result: [
+    W("x\'a b\'y", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 7,
+        line: 1,
+        col: 8,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap
new file mode 100644
index 0000000..8bb55b6
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-2.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$@\")?"
+---
+TokenizerResult(
+  input: "$@",
+  result: [
+    W("$@", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap
new file mode 100644
index 0000000..6d0ef12
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-3.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$!\")?"
+---
+TokenizerResult(
+  input: "$!",
+  result: [
+    W("$!", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap
new file mode 100644
index 0000000..2058d65
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-4.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$?\")?"
+---
+TokenizerResult(
+  input: "$?",
+  result: [
+    W("$?", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap
new file mode 100644
index 0000000..a76a319
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters-5.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$*\")?"
+---
+TokenizerResult(
+  input: "$*",
+  result: [
+    W("$*", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap
new file mode 100644
index 0000000..2a3d429
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_special_parameters.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$$\")?"
+---
+TokenizerResult(
+  input: "$$",
+  result: [
+    W("$$", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap
new file mode 100644
index 0000000..490dece
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion-2.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"a$x\")?"
+---
+TokenizerResult(
+  input: "a$x",
+  result: [
+    W("a$x", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap
new file mode 100644
index 0000000..158d342
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_unbraced_parameter_expansion.snap
@@ -0,0 +1,21 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"$x\")?"
+---
+TokenizerResult(
+  input: "$x",
+  result: [
+    W("$x", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+    )),
+  ],
+)
diff --git a/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap
new file mode 100644
index 0000000..af75b4a
--- /dev/null
+++ b/brush-parser/src/snapshots/brush_parser__tokenizer__tests__tokenize_whitespace.snap
@@ -0,0 +1,45 @@
+---
+source: brush-parser/src/tokenizer.rs
+expression: "test_tokenizer(\"1 2 3\")?"
+---
+TokenizerResult(
+  input: "1 2 3",
+  result: [
+    W("1", Loc(
+      start: Pos(
+        idx: 0,
+        line: 1,
+        col: 1,
+      ),
+      end: Pos(
+        idx: 1,
+        line: 1,
+        col: 2,
+      ),
+    )),
+    W("2", Loc(
+      start: Pos(
+        idx: 2,
+        line: 1,
+        col: 3,
+      ),
+      end: Pos(
+        idx: 3,
+        line: 1,
+        col: 4,
+      ),
+    )),
+    W("3", Loc(
+      start: Pos(
+        idx: 4,
+        line: 1,
+        col: 5,
+      ),
+      end: Pos(
+        idx: 5,
+        line: 1,
+        col: 6,
+      ),
+    )),
+  ],
+)