Repository: CrossGL/crosstl
Pull request #253 (2025-01-03 16:40:22)

Chore: Optimize backend Lexer Performance
Extension to all backends for the issue https://github.com/CrossGL/crosstl/issues/123 and PR https://github.com/CrossGL/crosstl/pull/249


Patch:
diff --git a/crosstl/backend/Metal/MetalLexer.py b/crosstl/backend/Metal/MetalLexer.py
index 8c0967d..963b22c 100644
--- a/crosstl/backend/Metal/MetalLexer.py
+++ b/crosstl/backend/Metal/MetalLexer.py
@@ -1,72 +1,78 @@
 import re
+from typing import Iterator, Tuple, List
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("PREPROCESSOR", r"#\w+"),
-    ("STRUCT", r"\bstruct\b"),
-    ("CONSTANT", r"\bconstant\b"),
-    ("TEXTURE2D", r"\btexture2d\b"),
-    ("buffer", r"\bbuffer\b"),
-    ("SAMPLER", r"\bsampler\b"),
-    ("VECTOR", r"\b(float|half|int|uint)[2-4]\b"),
-    ("FLOAT", r"\bfloat\b"),
-    ("HALF", r"\bhalf\b"),
-    ("INT", r"\bint\b"),
-    ("UINT", r"\buint\b"),
-    ("QUESTION", r"\?"),
-    ("BOOL", r"\bbool\b"),
-    ("VOID", r"\bvoid\b"),
-    ("RETURN", r"\breturn\b"),
-    ("ELSE_IF", r"\belse\s+if\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("KERNEL", r"\bkernel\b"),
-    ("VERTEX", r"\bvertex\b"),
-    ("FRAGMENT", r"\bfragment\b"),
-    ("USING", r"\busing\b"),
-    ("NAMESPACE", r"\bnamespace\b"),
-    ("METAL", r"\bmetal\b"),
-    ("DEVICE", r"\bdevice\b"),
-    ("THREADGROUP", r"\bthreadgroup\b"),
-    ("THREAD", r"\bthread\b"),
-    ("ATTRIBUTE", r"\[\[.*?\]\]"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("STRING", r'"[^"]*"'),
-    ("COMMA", r","),
-    ("COLON", r":"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("bitwise_and", r"&"),
-    ("WHITESPACE", r"\s+"),
-    ("MOD", r"%"),
-    ("ASSIGN_MOD", r"%="),
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("PREPROCESSOR", r"#\w+"),
+        ("STRUCT", r"\bstruct\b"),
+        ("CONSTANT", r"\bconstant\b"),
+        ("TEXTURE2D", r"\btexture2d\b"),
+        ("buffer", r"\bbuffer\b"),
+        ("SAMPLER", r"\bsampler\b"),
+        ("VECTOR", r"\b(float|half|int|uint)[2-4]\b"),
+        ("FLOAT", r"\bfloat\b"),
+        ("HALF", r"\bhalf\b"),
+        ("INT", r"\bint\b"),
+        ("UINT", r"\buint\b"),
+        ("QUESTION", r"\?"),
+        ("BOOL", r"\bbool\b"),
+        ("VOID", r"\bvoid\b"),
+        ("RETURN", r"\breturn\b"),
+        ("ELSE_IF", r"\belse\s+if\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("KERNEL", r"\bkernel\b"),
+        ("VERTEX", r"\bvertex\b"),
+        ("FRAGMENT", r"\bfragment\b"),
+        ("USING", r"\busing\b"),
+        ("NAMESPACE", r"\bnamespace\b"),
+        ("METAL", r"\bmetal\b"),
+        ("DEVICE", r"\bdevice\b"),
+        ("THREADGROUP", r"\bthreadgroup\b"),
+        ("THREAD", r"\bthread\b"),
+        ("ATTRIBUTE", r"\[\[.*?\]\]"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("STRING", r'"[^"]*"'),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("bitwise_and", r"&"),
+        ("WHITESPACE", r"\s+"),
+        ("MOD", r"%"),
+        ("ASSIGN_MOD", r"%="),
+    ]
+)
 
 KEYWORDS = {
     "struct": "STRUCT",
@@ -97,34 +103,46 @@ KEYWORDS = {
 
 
 class MetalLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
 
-        self.tokens.append(("EOF", ""))
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "MetalLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/Mojo/MojoLexer.py b/crosstl/backend/Mojo/MojoLexer.py
index b1395ec..4faa8f3 100644
--- a/crosstl/backend/Mojo/MojoLexer.py
+++ b/crosstl/backend/Mojo/MojoLexer.py
@@ -1,57 +1,63 @@
 import re
+from typing import Iterator, Tuple, List
+
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"#.*"),
+        ("COMMENT_MULTI", r'"""[\s\S]*?"""'),
+        ("STRUCT", r"\bstruct\b"),
+        ("LET", r"\blet\b"),
+        ("VAR", r"\bvar\b"),
+        ("FN", r"\bfn\b"),
+        ("RETURN", r"\breturn\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("WHILE", r"\bwhile\b"),
+        ("IMPORT", r"\bimport\b"),
+        ("DEF", r"\bdef\b"),
+        ("INT", r"\bInt\b"),
+        ("FLOAT", r"\bFloat\b"),
+        ("BOOL", r"\bBool\b"),
+        ("STRING", r"\bString\b"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("STRING_LITERAL", r'"[^"]*"'),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("WHITESPACE", r"\s+"),
+        ("MOD", r"%"),
+    ]
+)
 
-# Define the tokens for Mojo syntax
-TOKENS = [
-    ("COMMENT_SINGLE", r"#.*"),
-    ("COMMENT_MULTI", r'"""[\s\S]*?"""'),
-    ("STRUCT", r"\bstruct\b"),
-    ("LET", r"\blet\b"),
-    ("VAR", r"\bvar\b"),
-    ("FN", r"\bfn\b"),
-    ("RETURN", r"\breturn\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("WHILE", r"\bwhile\b"),
-    ("IMPORT", r"\bimport\b"),
-    ("DEF", r"\bdef\b"),
-    ("INT", r"\bInt\b"),
-    ("FLOAT", r"\bFloat\b"),
-    ("BOOL", r"\bBool\b"),
-    ("STRING", r"\bString\b"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("STRING_LITERAL", r'"[^"]*"'),
-    ("COMMA", r","),
-    ("COLON", r":"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("WHITESPACE", r"\s+"),
-    ("MOD", r"%"),
-]
 
 # Define keywords specific to mojo
 KEYWORDS = {
@@ -74,34 +80,46 @@ KEYWORDS = {
 
 
 class MojoLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
 
-    def tokenize(self):
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
+
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
 
-        self.tokens.append(("EOF", ""))
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "MojoLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/Opengl/OpenglLexer.py b/crosstl/backend/Opengl/OpenglLexer.py
index 5a1a18a..41f8f9f 100644
--- a/crosstl/backend/Opengl/OpenglLexer.py
+++ b/crosstl/backend/Opengl/OpenglLexer.py
@@ -1,78 +1,84 @@
 import re
+from typing import Iterator, Tuple, List
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("VERSION", r"#version"),
-    ("PREPROCESSOR", r"#\w+"),
-    ("CONSTANT", r"\bconst\b"),
-    ("STRUCT", r"\bstruct\b"),
-    ("UNIFORM", r"\buniform\b"),
-    ("SAMPLER2D", r"\bsampler2D\b"),
-    ("SAMPLERCUBE", r"\bsamplerCube\b"),
-    ("BUFFER", r"\bbuffer\b"),
-    ("VECTOR", r"\b(vec|ivec|uvec|bvec)[234]\b"),
-    ("MATRIX", r"\bmat[234](x[234])?\b"),
-    ("FLOAT", r"\bfloat\b"),
-    ("INT", r"\bint\b"),
-    ("DOUBLE", r"\bdouble\b"),
-    ("UINT", r"\buint\b"),
-    ("BOOL", r"\bbool\b"),
-    ("VOID", r"\bvoid\b"),
-    ("RETURN", r"\breturn\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("WHILE", r"\bwhile\b"),
-    ("DO", r"\bdo\b"),
-    ("IN", r"\bin\b"),
-    ("OUT", r"\bout\b"),
-    ("INOUT", r"\binout\b"),
-    ("LAYOUT", r"\blayout\b"),
-    ("ATTRIBUTE", r"\battribute\b"),
-    ("VARYING", r"\bvarying\b"),
-    ("CONST", r"\bconst\b"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?([eE][+-]?\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("STRING", r'"[^"]*"'),
-    ("COMMA", r","),
-    ("COLON", r":"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("ASSIGN_AND", r"&="),
-    ("ASSIGN_OR", r"\|="),
-    ("ASSIGN_XOR", r"\^="),
-    ("LOGICAL_AND", r"&&"),
-    ("LOGICAL_OR", r"\|\|"),
-    ("ASSIGN_MOD", r"%="),
-    ("MOD", r"%"),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("BITWISE_AND", r"&"),
-    ("BITWISE_OR", r"\|"),
-    ("BITWISE_XOR", r"\^"),
-    ("BITWISE_NOT", r"~"),
-    ("WHITESPACE", r"\s+"),
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("VERSION", r"#version"),
+        ("PREPROCESSOR", r"#\w+"),
+        ("CONSTANT", r"\bconst\b"),
+        ("STRUCT", r"\bstruct\b"),
+        ("UNIFORM", r"\buniform\b"),
+        ("SAMPLER2D", r"\bsampler2D\b"),
+        ("SAMPLERCUBE", r"\bsamplerCube\b"),
+        ("BUFFER", r"\bbuffer\b"),
+        ("VECTOR", r"\b(vec|ivec|uvec|bvec)[234]\b"),
+        ("MATRIX", r"\bmat[234](x[234])?\b"),
+        ("FLOAT", r"\bfloat\b"),
+        ("INT", r"\bint\b"),
+        ("DOUBLE", r"\bdouble\b"),
+        ("UINT", r"\buint\b"),
+        ("BOOL", r"\bbool\b"),
+        ("VOID", r"\bvoid\b"),
+        ("RETURN", r"\breturn\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("WHILE", r"\bwhile\b"),
+        ("DO", r"\bdo\b"),
+        ("IN", r"\bin\b"),
+        ("OUT", r"\bout\b"),
+        ("INOUT", r"\binout\b"),
+        ("LAYOUT", r"\blayout\b"),
+        ("ATTRIBUTE", r"\battribute\b"),
+        ("VARYING", r"\bvarying\b"),
+        ("CONST", r"\bconst\b"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?([eE][+-]?\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("STRING", r'"[^"]*"'),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("ASSIGN_AND", r"&="),
+        ("ASSIGN_OR", r"\|="),
+        ("ASSIGN_XOR", r"\^="),
+        ("LOGICAL_AND", r"&&"),
+        ("LOGICAL_OR", r"\|\|"),
+        ("ASSIGN_MOD", r"%="),
+        ("MOD", r"%"),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("BITWISE_AND", r"&"),
+        ("BITWISE_OR", r"\|"),
+        ("BITWISE_XOR", r"\^"),
+        ("BITWISE_NOT", r"~"),
+        ("WHITESPACE", r"\s+"),
+    ]
+)
 
 KEYWORDS = {
     "struct": "STRUCT",
@@ -103,33 +109,46 @@ KEYWORDS = {
 
 
 class GLSLLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
-        self.tokens.append(("EOF", ""))
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
+
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "GLSLLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/Vulkan/VulkanLexer.py b/crosstl/backend/Vulkan/VulkanLexer.py
index 4b2e5d8..b89f4d5 100644
--- a/crosstl/backend/Vulkan/VulkanLexer.py
+++ b/crosstl/backend/Vulkan/VulkanLexer.py
@@ -1,56 +1,62 @@
 import re
+from typing import Iterator, Tuple, List
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("WHITESPACE", r"\s+"),
-    ("SEMANTIC", r":\w+"),
-    ("PRE_INCREMENT", r"\+\+(?=\w)"),
-    ("PRE_DECREMENT", r"--(?=\w)"),
-    ("POST_INCREMENT", r"(?<=\w)\+\+"),
-    ("POST_DECREMENT", r"(?<=\w)--"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d*)?|\.\d+"),
-    ("SEMICOLON", r";"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("COMMA", r","),
-    ("DOT", r"\."),
-    ("EQUAL", r"=="),
-    ("ASSIGN_AND", r"&="),
-    ("ASSIGN_OR", r"\|="),
-    ("ASSIGN_XOR", r"\^="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("ASSIGN_MOD", r"%="),
-    ("ASSIGN_SHIFT_LEFT", r"<<="),
-    ("ASSIGN_SHIFT_RIGHT", r">>="),
-    ("BITWISE_SHIFT_LEFT", r"<<"),
-    ("BITWISE_SHIFT_RIGHT", r">>"),
-    ("EQUALS", r"="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("NOT_EQUAL", r"!="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("BINARY_AND", r"&"),
-    ("BINARY_OR", r"\|"),
-    ("BINARY_XOR", r"\^"),
-    ("BINARY_NOT", r"~"),
-    ("QUESTION", r"\?"),
-    ("COLON", r":"),
-    ("MOD", r"%"),
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("WHITESPACE", r"\s+"),
+        ("SEMANTIC", r":\w+"),
+        ("PRE_INCREMENT", r"\+\+(?=\w)"),
+        ("PRE_DECREMENT", r"--(?=\w)"),
+        ("POST_INCREMENT", r"(?<=\w)\+\+"),
+        ("POST_DECREMENT", r"(?<=\w)--"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d*)?|\.\d+"),
+        ("SEMICOLON", r";"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("COMMA", r","),
+        ("DOT", r"\."),
+        ("EQUAL", r"=="),
+        ("ASSIGN_AND", r"&="),
+        ("ASSIGN_OR", r"\|="),
+        ("ASSIGN_XOR", r"\^="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("ASSIGN_MOD", r"%="),
+        ("ASSIGN_SHIFT_LEFT", r"<<="),
+        ("ASSIGN_SHIFT_RIGHT", r">>="),
+        ("BITWISE_SHIFT_LEFT", r"<<"),
+        ("BITWISE_SHIFT_RIGHT", r">>"),
+        ("EQUALS", r"="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("NOT_EQUAL", r"!="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("BINARY_AND", r"&"),
+        ("BINARY_OR", r"\|"),
+        ("BINARY_XOR", r"\^"),
+        ("BINARY_NOT", r"~"),
+        ("QUESTION", r"\?"),
+        ("COLON", r":"),
+        ("MOD", r"%"),
+    ]
+)
 
 KEYWORDS = {
     "struct": "STRUCT",
@@ -138,40 +144,46 @@ VALID_DATA_TYPES = [
 
 
 class VulkanLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type == "VERSION":
-                        self.tokens.append((token_type, text))
-                    elif token_type == "VERSION_NUMBER":
-                        self.tokens.append((token_type, text))
-                    elif token_type == "CORE":
-                        self.tokens.append((token_type, text))
-                    elif token_type != "WHITESPACE":  # Ignore whitespace tokens
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
-                unmatched_char = self.code[pos]
-                highlighted_code = (
-                    self.code[:pos] + "[" + self.code[pos] + "]" + self.code[pos + 1 :]
-                )
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
-                    f"Illegal character '{unmatched_char}' at position {pos}\n{highlighted_code}"
+                    f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
 
-        self.tokens.append(("EOF", None))
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "VulkanLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/slang/SlangLexer.py b/crosstl/backend/slang/SlangLexer.py
index 9d8808b..44dc39f 100644
--- a/crosstl/backend/slang/SlangLexer.py
+++ b/crosstl/backend/slang/SlangLexer.py
@@ -1,80 +1,83 @@
 import re
+from typing import Iterator, Tuple, List
 
-# Token definitions
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("STRUCT", r"\bstruct\b"),
-    ("CBUFFER", r"\bcbuffer\b"),
-    ("TYPE_SHADER", r'\[shader\("(vertex|fragment|compute)"\)\]'),
-    ("SHADER", r"\bshader\b"),
-    ("STRING", r'"(?:\\.|[^"\\])*"'),
-    ("TEXTURE2D", r"\bTexture2D\b"),
-    ("SAMPLER_STATE", r"\bSamplerState\b"),
-    ("FVECTOR", r"\bfloat[2-4]\b"),
-    ("FLOAT", r"\bfloat\b"),
-    ("INT", r"\bint\b"),
-    ("UINT", r"\buint\b"),
-    ("BOOL", r"\bbool\b"),
-    ("MATRIX", r"\bfloat[2-4]x[2-4]\b"),
-    ("VOID", r"\bvoid\b"),
-    ("RETURN", r"\breturn\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE_IF", r"\belse\s+if\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("WHILE", r"\bwhile\b"),
-    ("DO", r"\bdo\b"),
-    ("SWITCH", r"\bswitch\b"),
-    ("CASE", r"\bcase\b"),
-    ("DEFAULT", r"\bdefault\b"),
-    ("BREAK", r"\bbreak\b"),
-    ("CONTINUE", r"\bcontinue\b"),
-    ("REGISTER", r"\bregister\b"),
-    ("STRING", r'"[^"]*"'),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("COMMA", r","),
-    ("COLON", r":"),  # Separate token for single colon, if needed separately
-    ("QUESTION", r"\?"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("DOT", r"\."),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("EQUALS", r"="),
-    ("WHITESPACE", r"\s+"),
-    # Slang-specific tokens
-    ("IMPORT", r"\bimport\b"),
-    ("EXPORT", r"\bexport\b"),
-    ("GENERIC", r"\b__generic\b"),
-    ("EXTENSION", r"\bextension\b"),
-    ("TYPEDEF", r"\btypedef\b"),
-    ("CONSTEXPR", r"\bconstexpr\b"),
-    ("STATIC", r"\bstatic\b"),
-    ("INLINE", r"\binline\b"),
-    ("MOD", r"%"),  # Add modulus operator
-    # Add modulus assignment
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("STRUCT", r"\bstruct\b"),
+        ("CBUFFER", r"\bcbuffer\b"),
+        ("TYPE_SHADER", r'\[shader\("(vertex|fragment|compute)"\)\]'),
+        ("SHADER", r"\bshader\b"),
+        ("STRING", r'"(?:\\.|[^"\\])*"'),
+        ("TEXTURE2D", r"\bTexture2D\b"),
+        ("SAMPLER_STATE", r"\bSamplerState\b"),
+        ("FVECTOR", r"\bfloat[2-4]\b"),
+        ("FLOAT", r"\bfloat\b"),
+        ("INT", r"\bint\b"),
+        ("UINT", r"\buint\b"),
+        ("BOOL", r"\bbool\b"),
+        ("MATRIX", r"\bfloat[2-4]x[2-4]\b"),
+        ("VOID", r"\bvoid\b"),
+        ("RETURN", r"\breturn\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE_IF", r"\belse\s+if\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("WHILE", r"\bwhile\b"),
+        ("DO", r"\bdo\b"),
+        ("SWITCH", r"\bswitch\b"),
+        ("CASE", r"\bcase\b"),
+        ("DEFAULT", r"\bdefault\b"),
+        ("BREAK", r"\bbreak\b"),
+        ("CONTINUE", r"\bcontinue\b"),
+        ("REGISTER", r"\bregister\b"),
+        ("STRING", r'"[^"]*"'),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("QUESTION", r"\?"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("DOT", r"\."),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("EQUALS", r"="),
+        ("WHITESPACE", r"\s+"),
+        ("IMPORT", r"\bimport\b"),
+        ("EXPORT", r"\bexport\b"),
+        ("GENERIC", r"\b__generic\b"),
+        ("EXTENSION", r"\bextension\b"),
+        ("TYPEDEF", r"\btypedef\b"),
+        ("CONSTEXPR", r"\bconstexpr\b"),
+        ("STATIC", r"\bstatic\b"),
+        ("INLINE", r"\binline\b"),
+        ("MOD", r"%"),
+    ]
+)
 
 # Keywords map for matching identifiers to token types
 KEYWORDS = {
@@ -114,33 +117,46 @@ KEYWORDS = {
 
 
 class SlangLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text.strip())
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
-        self.tokens.append(("EOF", ""))
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
+
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "SlangLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())


Test patch:
diff --git a/tests/test_backend/test_metal/test_codegen.py b/tests/test_backend/test_metal/test_codegen.py
index 72e4f74..12a0426 100644
--- a/tests/test_backend/test_metal/test_codegen.py
+++ b/tests/test_backend/test_metal/test_codegen.py
@@ -8,7 +8,7 @@ from crosstl.backend.Metal.MetalCrossGLCodeGen import MetalToCrossGLConverter
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MetalLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_metal/test_lexer.py b/tests/test_backend/test_metal/test_lexer.py
index e745ca7..8ea7b83 100644
--- a/tests/test_backend/test_metal/test_lexer.py
+++ b/tests/test_backend/test_metal/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.Metal.MetalLexer import MetalLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MetalLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_struct_tokenization():
diff --git a/tests/test_backend/test_metal/test_parser.py b/tests/test_backend/test_metal/test_parser.py
index 02286b7..15b2f6f 100644
--- a/tests/test_backend/test_metal/test_parser.py
+++ b/tests/test_backend/test_metal/test_parser.py
@@ -7,7 +7,7 @@ from crosstl.backend.Metal.MetalParser import MetalParser
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MetalLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_mojo/test_codegen.py b/tests/test_backend/test_mojo/test_codegen.py
index 173f817..7ca911b 100644
--- a/tests/test_backend/test_mojo/test_codegen.py
+++ b/tests/test_backend/test_mojo/test_codegen.py
@@ -16,7 +16,7 @@ def generate_code(ast_node):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MojoLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_mojo/test_lexer.py b/tests/test_backend/test_mojo/test_lexer.py
index e23afce..95f7d18 100644
--- a/tests/test_backend/test_mojo/test_lexer.py
+++ b/tests/test_backend/test_mojo/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.Mojo.MojoLexer import MojoLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MojoLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_tokenization():
diff --git a/tests/test_backend/test_mojo/test_parser.py b/tests/test_backend/test_mojo/test_parser.py
index 295781b..4759ac1 100644
--- a/tests/test_backend/test_mojo/test_parser.py
+++ b/tests/test_backend/test_mojo/test_parser.py
@@ -19,7 +19,7 @@ def parse_code(tokens: List):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MojoLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_parsing():
diff --git a/tests/test_backend/test_opengl/test_codegen.py b/tests/test_backend/test_opengl/test_codegen.py
index bbb83b3..760bfe0 100644
--- a/tests/test_backend/test_opengl/test_codegen.py
+++ b/tests/test_backend/test_opengl/test_codegen.py
@@ -16,7 +16,7 @@ def tokenize_code(code: str) -> List:
 
     """
     lexer = GLSLLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(Tokens: List, shader_type="vertex") -> List:
diff --git a/tests/test_backend/test_opengl/test_lexer.py b/tests/test_backend/test_opengl/test_lexer.py
index a112929..5226520 100644
--- a/tests/test_backend/test_opengl/test_lexer.py
+++ b/tests/test_backend/test_opengl/test_lexer.py
@@ -14,7 +14,7 @@ def tokenize_code(code: str) -> List:
 
     """
     lexer = GLSLLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_input_output_tokenization():
diff --git a/tests/test_backend/test_opengl/test_parser.py b/tests/test_backend/test_opengl/test_parser.py
index 79a107b..5951406 100644
--- a/tests/test_backend/test_opengl/test_parser.py
+++ b/tests/test_backend/test_opengl/test_parser.py
@@ -15,7 +15,7 @@ def tokenize_code(code: str) -> List:
 
     """
     lexer = GLSLLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(Tokens: List, shader_type="vertex") -> List:
diff --git a/tests/test_backend/test_slang/test_codegen.py b/tests/test_backend/test_slang/test_codegen.py
index 6a00430..cb5387f 100644
--- a/tests/test_backend/test_slang/test_codegen.py
+++ b/tests/test_backend/test_slang/test_codegen.py
@@ -19,7 +19,7 @@ def generate_code(ast_node):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = SlangLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_slang/test_lexer.py b/tests/test_backend/test_slang/test_lexer.py
index a9bbd98..1ef8bd2 100644
--- a/tests/test_backend/test_slang/test_lexer.py
+++ b/tests/test_backend/test_slang/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.slang.SlangLexer import SlangLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = SlangLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_struct_tokenization():
diff --git a/tests/test_backend/test_slang/test_parser.py b/tests/test_backend/test_slang/test_parser.py
index 69fd232..742f597 100644
--- a/tests/test_backend/test_slang/test_parser.py
+++ b/tests/test_backend/test_slang/test_parser.py
@@ -19,7 +19,7 @@ def parse_code(tokens: List):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = SlangLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_struct_parsing():
diff --git a/tests/test_backend/test_vulkan/test_codegen.py b/tests/test_backend/test_vulkan/test_codegen.py
index 6e2c91c..71a274d 100644
--- a/tests/test_backend/test_vulkan/test_codegen.py
+++ b/tests/test_backend/test_vulkan/test_codegen.py
@@ -16,7 +16,7 @@ def generate_code(ast_node):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = VulkanLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_vulkan/test_lexer.py b/tests/test_backend/test_vulkan/test_lexer.py
index 9fabf09..e68db35 100644
--- a/tests/test_backend/test_vulkan/test_lexer.py
+++ b/tests/test_backend/test_vulkan/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.Vulkan.VulkanLexer import VulkanLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = VulkanLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_tokenization():
diff --git a/tests/test_backend/test_vulkan/test_parser.py b/tests/test_backend/test_vulkan/test_parser.py
index d1af630..ec7789a 100644
--- a/tests/test_backend/test_vulkan/test_parser.py
+++ b/tests/test_backend/test_vulkan/test_parser.py
@@ -19,7 +19,7 @@ def parse_code(tokens: List):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = VulkanLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_parsing():