Repository: CrossGL/crosstl
Pull request #252 (2025-01-02 16:52:53)

Refactor code generation logic for improved readability and maintaina…
## Summary

Refactors code generation logic to improve readability and maintainability by replacing redundant conditional checks with more concise approaches, enhancing code clarity and structure.


### Changes
- Replaced repeated shader type checks in `generate()` with a dictionary-based approach, improving clarity and reducing redundancy.
- Simplified the `generate_if()` method to handle `else_if_conditions` and `else_if_bodies` more clearly, enhancing readability.
- Refactored `generate_statement()` by using a dictionary-based mapping of node types to their corresponding handlers, reducing conditional branching and improving code organization.

### Testing

The changes were tested by running the code through existing test cases.
Manually tested in the local environment to ensure no functionality was broken.

### Checklist

- [ ] Tests cover new or changed code (or reason why not)
- [ ] Linked the issue with a closing keyword (if applicable)
- [ ] Everything builds and runs locally


Vruddhi18: Please review this @NripeshN 

Patch:
diff --git a/crosstl/backend/Metal/MetalLexer.py b/crosstl/backend/Metal/MetalLexer.py
index 8c0967d..963b22c 100644
--- a/crosstl/backend/Metal/MetalLexer.py
+++ b/crosstl/backend/Metal/MetalLexer.py
@@ -1,72 +1,78 @@
 import re
+from typing import Iterator, Tuple, List
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("PREPROCESSOR", r"#\w+"),
-    ("STRUCT", r"\bstruct\b"),
-    ("CONSTANT", r"\bconstant\b"),
-    ("TEXTURE2D", r"\btexture2d\b"),
-    ("buffer", r"\bbuffer\b"),
-    ("SAMPLER", r"\bsampler\b"),
-    ("VECTOR", r"\b(float|half|int|uint)[2-4]\b"),
-    ("FLOAT", r"\bfloat\b"),
-    ("HALF", r"\bhalf\b"),
-    ("INT", r"\bint\b"),
-    ("UINT", r"\buint\b"),
-    ("QUESTION", r"\?"),
-    ("BOOL", r"\bbool\b"),
-    ("VOID", r"\bvoid\b"),
-    ("RETURN", r"\breturn\b"),
-    ("ELSE_IF", r"\belse\s+if\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("KERNEL", r"\bkernel\b"),
-    ("VERTEX", r"\bvertex\b"),
-    ("FRAGMENT", r"\bfragment\b"),
-    ("USING", r"\busing\b"),
-    ("NAMESPACE", r"\bnamespace\b"),
-    ("METAL", r"\bmetal\b"),
-    ("DEVICE", r"\bdevice\b"),
-    ("THREADGROUP", r"\bthreadgroup\b"),
-    ("THREAD", r"\bthread\b"),
-    ("ATTRIBUTE", r"\[\[.*?\]\]"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("STRING", r'"[^"]*"'),
-    ("COMMA", r","),
-    ("COLON", r":"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("bitwise_and", r"&"),
-    ("WHITESPACE", r"\s+"),
-    ("MOD", r"%"),
-    ("ASSIGN_MOD", r"%="),
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("PREPROCESSOR", r"#\w+"),
+        ("STRUCT", r"\bstruct\b"),
+        ("CONSTANT", r"\bconstant\b"),
+        ("TEXTURE2D", r"\btexture2d\b"),
+        ("buffer", r"\bbuffer\b"),
+        ("SAMPLER", r"\bsampler\b"),
+        ("VECTOR", r"\b(float|half|int|uint)[2-4]\b"),
+        ("FLOAT", r"\bfloat\b"),
+        ("HALF", r"\bhalf\b"),
+        ("INT", r"\bint\b"),
+        ("UINT", r"\buint\b"),
+        ("QUESTION", r"\?"),
+        ("BOOL", r"\bbool\b"),
+        ("VOID", r"\bvoid\b"),
+        ("RETURN", r"\breturn\b"),
+        ("ELSE_IF", r"\belse\s+if\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("KERNEL", r"\bkernel\b"),
+        ("VERTEX", r"\bvertex\b"),
+        ("FRAGMENT", r"\bfragment\b"),
+        ("USING", r"\busing\b"),
+        ("NAMESPACE", r"\bnamespace\b"),
+        ("METAL", r"\bmetal\b"),
+        ("DEVICE", r"\bdevice\b"),
+        ("THREADGROUP", r"\bthreadgroup\b"),
+        ("THREAD", r"\bthread\b"),
+        ("ATTRIBUTE", r"\[\[.*?\]\]"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("STRING", r'"[^"]*"'),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("bitwise_and", r"&"),
+        ("WHITESPACE", r"\s+"),
+        ("MOD", r"%"),
+        ("ASSIGN_MOD", r"%="),
+    ]
+)
 
 KEYWORDS = {
     "struct": "STRUCT",
@@ -97,34 +103,46 @@ KEYWORDS = {
 
 
 class MetalLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
 
-        self.tokens.append(("EOF", ""))
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "MetalLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/Mojo/MojoLexer.py b/crosstl/backend/Mojo/MojoLexer.py
index b1395ec..4faa8f3 100644
--- a/crosstl/backend/Mojo/MojoLexer.py
+++ b/crosstl/backend/Mojo/MojoLexer.py
@@ -1,57 +1,63 @@
 import re
+from typing import Iterator, Tuple, List
+
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"#.*"),
+        ("COMMENT_MULTI", r'"""[\s\S]*?"""'),
+        ("STRUCT", r"\bstruct\b"),
+        ("LET", r"\blet\b"),
+        ("VAR", r"\bvar\b"),
+        ("FN", r"\bfn\b"),
+        ("RETURN", r"\breturn\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("WHILE", r"\bwhile\b"),
+        ("IMPORT", r"\bimport\b"),
+        ("DEF", r"\bdef\b"),
+        ("INT", r"\bInt\b"),
+        ("FLOAT", r"\bFloat\b"),
+        ("BOOL", r"\bBool\b"),
+        ("STRING", r"\bString\b"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("STRING_LITERAL", r'"[^"]*"'),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("WHITESPACE", r"\s+"),
+        ("MOD", r"%"),
+    ]
+)
 
-# Define the tokens for Mojo syntax
-TOKENS = [
-    ("COMMENT_SINGLE", r"#.*"),
-    ("COMMENT_MULTI", r'"""[\s\S]*?"""'),
-    ("STRUCT", r"\bstruct\b"),
-    ("LET", r"\blet\b"),
-    ("VAR", r"\bvar\b"),
-    ("FN", r"\bfn\b"),
-    ("RETURN", r"\breturn\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("WHILE", r"\bwhile\b"),
-    ("IMPORT", r"\bimport\b"),
-    ("DEF", r"\bdef\b"),
-    ("INT", r"\bInt\b"),
-    ("FLOAT", r"\bFloat\b"),
-    ("BOOL", r"\bBool\b"),
-    ("STRING", r"\bString\b"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("STRING_LITERAL", r'"[^"]*"'),
-    ("COMMA", r","),
-    ("COLON", r":"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("WHITESPACE", r"\s+"),
-    ("MOD", r"%"),
-]
 
 # Define keywords specific to mojo
 KEYWORDS = {
@@ -74,34 +80,46 @@ KEYWORDS = {
 
 
 class MojoLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
 
-    def tokenize(self):
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
+
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
 
-        self.tokens.append(("EOF", ""))
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "MojoLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/Opengl/OpenglLexer.py b/crosstl/backend/Opengl/OpenglLexer.py
index 5a1a18a..41f8f9f 100644
--- a/crosstl/backend/Opengl/OpenglLexer.py
+++ b/crosstl/backend/Opengl/OpenglLexer.py
@@ -1,78 +1,84 @@
 import re
+from typing import Iterator, Tuple, List
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("VERSION", r"#version"),
-    ("PREPROCESSOR", r"#\w+"),
-    ("CONSTANT", r"\bconst\b"),
-    ("STRUCT", r"\bstruct\b"),
-    ("UNIFORM", r"\buniform\b"),
-    ("SAMPLER2D", r"\bsampler2D\b"),
-    ("SAMPLERCUBE", r"\bsamplerCube\b"),
-    ("BUFFER", r"\bbuffer\b"),
-    ("VECTOR", r"\b(vec|ivec|uvec|bvec)[234]\b"),
-    ("MATRIX", r"\bmat[234](x[234])?\b"),
-    ("FLOAT", r"\bfloat\b"),
-    ("INT", r"\bint\b"),
-    ("DOUBLE", r"\bdouble\b"),
-    ("UINT", r"\buint\b"),
-    ("BOOL", r"\bbool\b"),
-    ("VOID", r"\bvoid\b"),
-    ("RETURN", r"\breturn\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("WHILE", r"\bwhile\b"),
-    ("DO", r"\bdo\b"),
-    ("IN", r"\bin\b"),
-    ("OUT", r"\bout\b"),
-    ("INOUT", r"\binout\b"),
-    ("LAYOUT", r"\blayout\b"),
-    ("ATTRIBUTE", r"\battribute\b"),
-    ("VARYING", r"\bvarying\b"),
-    ("CONST", r"\bconst\b"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?([eE][+-]?\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("STRING", r'"[^"]*"'),
-    ("COMMA", r","),
-    ("COLON", r":"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("ASSIGN_AND", r"&="),
-    ("ASSIGN_OR", r"\|="),
-    ("ASSIGN_XOR", r"\^="),
-    ("LOGICAL_AND", r"&&"),
-    ("LOGICAL_OR", r"\|\|"),
-    ("ASSIGN_MOD", r"%="),
-    ("MOD", r"%"),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("BITWISE_AND", r"&"),
-    ("BITWISE_OR", r"\|"),
-    ("BITWISE_XOR", r"\^"),
-    ("BITWISE_NOT", r"~"),
-    ("WHITESPACE", r"\s+"),
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("VERSION", r"#version"),
+        ("PREPROCESSOR", r"#\w+"),
+        ("CONSTANT", r"\bconst\b"),
+        ("STRUCT", r"\bstruct\b"),
+        ("UNIFORM", r"\buniform\b"),
+        ("SAMPLER2D", r"\bsampler2D\b"),
+        ("SAMPLERCUBE", r"\bsamplerCube\b"),
+        ("BUFFER", r"\bbuffer\b"),
+        ("VECTOR", r"\b(vec|ivec|uvec|bvec)[234]\b"),
+        ("MATRIX", r"\bmat[234](x[234])?\b"),
+        ("FLOAT", r"\bfloat\b"),
+        ("INT", r"\bint\b"),
+        ("DOUBLE", r"\bdouble\b"),
+        ("UINT", r"\buint\b"),
+        ("BOOL", r"\bbool\b"),
+        ("VOID", r"\bvoid\b"),
+        ("RETURN", r"\breturn\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("WHILE", r"\bwhile\b"),
+        ("DO", r"\bdo\b"),
+        ("IN", r"\bin\b"),
+        ("OUT", r"\bout\b"),
+        ("INOUT", r"\binout\b"),
+        ("LAYOUT", r"\blayout\b"),
+        ("ATTRIBUTE", r"\battribute\b"),
+        ("VARYING", r"\bvarying\b"),
+        ("CONST", r"\bconst\b"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?([eE][+-]?\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("STRING", r'"[^"]*"'),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("ASSIGN_AND", r"&="),
+        ("ASSIGN_OR", r"\|="),
+        ("ASSIGN_XOR", r"\^="),
+        ("LOGICAL_AND", r"&&"),
+        ("LOGICAL_OR", r"\|\|"),
+        ("ASSIGN_MOD", r"%="),
+        ("MOD", r"%"),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("BITWISE_AND", r"&"),
+        ("BITWISE_OR", r"\|"),
+        ("BITWISE_XOR", r"\^"),
+        ("BITWISE_NOT", r"~"),
+        ("WHITESPACE", r"\s+"),
+    ]
+)
 
 KEYWORDS = {
     "struct": "STRUCT",
@@ -103,33 +109,46 @@ KEYWORDS = {
 
 
 class GLSLLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
-        self.tokens.append(("EOF", ""))
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
+
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "GLSLLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/Vulkan/VulkanLexer.py b/crosstl/backend/Vulkan/VulkanLexer.py
index 4b2e5d8..b89f4d5 100644
--- a/crosstl/backend/Vulkan/VulkanLexer.py
+++ b/crosstl/backend/Vulkan/VulkanLexer.py
@@ -1,56 +1,62 @@
 import re
+from typing import Iterator, Tuple, List
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("WHITESPACE", r"\s+"),
-    ("SEMANTIC", r":\w+"),
-    ("PRE_INCREMENT", r"\+\+(?=\w)"),
-    ("PRE_DECREMENT", r"--(?=\w)"),
-    ("POST_INCREMENT", r"(?<=\w)\+\+"),
-    ("POST_DECREMENT", r"(?<=\w)--"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d*)?|\.\d+"),
-    ("SEMICOLON", r";"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("COMMA", r","),
-    ("DOT", r"\."),
-    ("EQUAL", r"=="),
-    ("ASSIGN_AND", r"&="),
-    ("ASSIGN_OR", r"\|="),
-    ("ASSIGN_XOR", r"\^="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("ASSIGN_MOD", r"%="),
-    ("ASSIGN_SHIFT_LEFT", r"<<="),
-    ("ASSIGN_SHIFT_RIGHT", r">>="),
-    ("BITWISE_SHIFT_LEFT", r"<<"),
-    ("BITWISE_SHIFT_RIGHT", r">>"),
-    ("EQUALS", r"="),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("NOT_EQUAL", r"!="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("BINARY_AND", r"&"),
-    ("BINARY_OR", r"\|"),
-    ("BINARY_XOR", r"\^"),
-    ("BINARY_NOT", r"~"),
-    ("QUESTION", r"\?"),
-    ("COLON", r":"),
-    ("MOD", r"%"),
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("WHITESPACE", r"\s+"),
+        ("SEMANTIC", r":\w+"),
+        ("PRE_INCREMENT", r"\+\+(?=\w)"),
+        ("PRE_DECREMENT", r"--(?=\w)"),
+        ("POST_INCREMENT", r"(?<=\w)\+\+"),
+        ("POST_DECREMENT", r"(?<=\w)--"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d*)?|\.\d+"),
+        ("SEMICOLON", r";"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("COMMA", r","),
+        ("DOT", r"\."),
+        ("EQUAL", r"=="),
+        ("ASSIGN_AND", r"&="),
+        ("ASSIGN_OR", r"\|="),
+        ("ASSIGN_XOR", r"\^="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("ASSIGN_MOD", r"%="),
+        ("ASSIGN_SHIFT_LEFT", r"<<="),
+        ("ASSIGN_SHIFT_RIGHT", r">>="),
+        ("BITWISE_SHIFT_LEFT", r"<<"),
+        ("BITWISE_SHIFT_RIGHT", r">>"),
+        ("EQUALS", r"="),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("NOT_EQUAL", r"!="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("BINARY_AND", r"&"),
+        ("BINARY_OR", r"\|"),
+        ("BINARY_XOR", r"\^"),
+        ("BINARY_NOT", r"~"),
+        ("QUESTION", r"\?"),
+        ("COLON", r":"),
+        ("MOD", r"%"),
+    ]
+)
 
 KEYWORDS = {
     "struct": "STRUCT",
@@ -138,40 +144,46 @@ VALID_DATA_TYPES = [
 
 
 class VulkanLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type == "VERSION":
-                        self.tokens.append((token_type, text))
-                    elif token_type == "VERSION_NUMBER":
-                        self.tokens.append((token_type, text))
-                    elif token_type == "CORE":
-                        self.tokens.append((token_type, text))
-                    elif token_type != "WHITESPACE":  # Ignore whitespace tokens
-                        token = (token_type, text)
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
-                unmatched_char = self.code[pos]
-                highlighted_code = (
-                    self.code[:pos] + "[" + self.code[pos] + "]" + self.code[pos + 1 :]
-                )
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
-                    f"Illegal character '{unmatched_char}' at position {pos}\n{highlighted_code}"
+                    f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
 
-        self.tokens.append(("EOF", None))
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "VulkanLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/backend/slang/SlangLexer.py b/crosstl/backend/slang/SlangLexer.py
index 9d8808b..44dc39f 100644
--- a/crosstl/backend/slang/SlangLexer.py
+++ b/crosstl/backend/slang/SlangLexer.py
@@ -1,80 +1,83 @@
 import re
+from typing import Iterator, Tuple, List
 
-# Token definitions
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("STRUCT", r"\bstruct\b"),
-    ("CBUFFER", r"\bcbuffer\b"),
-    ("TYPE_SHADER", r'\[shader\("(vertex|fragment|compute)"\)\]'),
-    ("SHADER", r"\bshader\b"),
-    ("STRING", r'"(?:\\.|[^"\\])*"'),
-    ("TEXTURE2D", r"\bTexture2D\b"),
-    ("SAMPLER_STATE", r"\bSamplerState\b"),
-    ("FVECTOR", r"\bfloat[2-4]\b"),
-    ("FLOAT", r"\bfloat\b"),
-    ("INT", r"\bint\b"),
-    ("UINT", r"\buint\b"),
-    ("BOOL", r"\bbool\b"),
-    ("MATRIX", r"\bfloat[2-4]x[2-4]\b"),
-    ("VOID", r"\bvoid\b"),
-    ("RETURN", r"\breturn\b"),
-    ("IF", r"\bif\b"),
-    ("ELSE_IF", r"\belse\s+if\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("WHILE", r"\bwhile\b"),
-    ("DO", r"\bdo\b"),
-    ("SWITCH", r"\bswitch\b"),
-    ("CASE", r"\bcase\b"),
-    ("DEFAULT", r"\bdefault\b"),
-    ("BREAK", r"\bbreak\b"),
-    ("CONTINUE", r"\bcontinue\b"),
-    ("REGISTER", r"\bregister\b"),
-    ("STRING", r'"[^"]*"'),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("LBRACKET", r"\["),
-    ("RBRACKET", r"\]"),
-    ("SEMICOLON", r";"),
-    ("COMMA", r","),
-    ("COLON", r":"),  # Separate token for single colon, if needed separately
-    ("QUESTION", r"\?"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("LESS_THAN", r"<"),
-    ("GREATER_THAN", r">"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("PLUS_EQUALS", r"\+="),
-    ("MINUS_EQUALS", r"-="),
-    ("MULTIPLY_EQUALS", r"\*="),
-    ("DIVIDE_EQUALS", r"/="),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
-    ("DOT", r"\."),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("EQUALS", r"="),
-    ("WHITESPACE", r"\s+"),
-    # Slang-specific tokens
-    ("IMPORT", r"\bimport\b"),
-    ("EXPORT", r"\bexport\b"),
-    ("GENERIC", r"\b__generic\b"),
-    ("EXTENSION", r"\bextension\b"),
-    ("TYPEDEF", r"\btypedef\b"),
-    ("CONSTEXPR", r"\bconstexpr\b"),
-    ("STATIC", r"\bstatic\b"),
-    ("INLINE", r"\binline\b"),
-    ("MOD", r"%"),  # Add modulus operator
-    # Add modulus assignment
-]
+# using sets for faster lookup
+SKIP_TOKENS = {"WHITESPACE", "COMMENT_SINGLE", "COMMENT_MULTI"}
+
+TOKENS = tuple(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("STRUCT", r"\bstruct\b"),
+        ("CBUFFER", r"\bcbuffer\b"),
+        ("TYPE_SHADER", r'\[shader\("(vertex|fragment|compute)"\)\]'),
+        ("SHADER", r"\bshader\b"),
+        ("STRING", r'"(?:\\.|[^"\\])*"'),
+        ("TEXTURE2D", r"\bTexture2D\b"),
+        ("SAMPLER_STATE", r"\bSamplerState\b"),
+        ("FVECTOR", r"\bfloat[2-4]\b"),
+        ("FLOAT", r"\bfloat\b"),
+        ("INT", r"\bint\b"),
+        ("UINT", r"\buint\b"),
+        ("BOOL", r"\bbool\b"),
+        ("MATRIX", r"\bfloat[2-4]x[2-4]\b"),
+        ("VOID", r"\bvoid\b"),
+        ("RETURN", r"\breturn\b"),
+        ("IF", r"\bif\b"),
+        ("ELSE_IF", r"\belse\s+if\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("WHILE", r"\bwhile\b"),
+        ("DO", r"\bdo\b"),
+        ("SWITCH", r"\bswitch\b"),
+        ("CASE", r"\bcase\b"),
+        ("DEFAULT", r"\bdefault\b"),
+        ("BREAK", r"\bbreak\b"),
+        ("CONTINUE", r"\bcontinue\b"),
+        ("REGISTER", r"\bregister\b"),
+        ("STRING", r'"[^"]*"'),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("LBRACKET", r"\["),
+        ("RBRACKET", r"\]"),
+        ("SEMICOLON", r";"),
+        ("COMMA", r","),
+        ("COLON", r":"),
+        ("QUESTION", r"\?"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("LESS_THAN", r"<"),
+        ("GREATER_THAN", r">"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("PLUS_EQUALS", r"\+="),
+        ("MINUS_EQUALS", r"-="),
+        ("MULTIPLY_EQUALS", r"\*="),
+        ("DIVIDE_EQUALS", r"/="),
+        ("AND", r"&&"),
+        ("OR", r"\|\|"),
+        ("DOT", r"\."),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("EQUALS", r"="),
+        ("WHITESPACE", r"\s+"),
+        ("IMPORT", r"\bimport\b"),
+        ("EXPORT", r"\bexport\b"),
+        ("GENERIC", r"\b__generic\b"),
+        ("EXTENSION", r"\bextension\b"),
+        ("TYPEDEF", r"\btypedef\b"),
+        ("CONSTEXPR", r"\bconstexpr\b"),
+        ("STATIC", r"\bstatic\b"),
+        ("INLINE", r"\binline\b"),
+        ("MOD", r"%"),
+    ]
+)
 
 # Keywords map for matching identifiers to token types
 KEYWORDS = {
@@ -114,33 +117,46 @@ KEYWORDS = {
 
 
 class SlangLexer:
-    def __init__(self, code):
+    def __init__(self, code: str):
+        self._token_patterns = [(name, re.compile(pattern)) for name, pattern in TOKENS]
         self.code = code
-        self.tokens = []
-        self.tokenize()
+        self._length = len(code)
+
+    def tokenize(self) -> List[Tuple[str, str]]:
+        # tokenize the input code and return list of tokens
+        return list(self.token_generator())
 
-    def tokenize(self):
+    def token_generator(self) -> Iterator[Tuple[str, str]]:
+        # function that yields tokens one at a time
         pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type not in [
-                        "WHITESPACE",
-                        "COMMENT_SINGLE",
-                        "COMMENT_MULTI",
-                    ]:
-                        token = (token_type, text.strip())
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
+        while pos < self._length:
+            token = self._next_token(pos)
+            if token is None:
                 raise SyntaxError(
                     f"Illegal character '{self.code[pos]}' at position {pos}"
                 )
-        self.tokens.append(("EOF", ""))
+            new_pos, token_type, text = token
+
+            if token_type == "IDENTIFIER" and text in KEYWORDS:
+                token_type = KEYWORDS[text]
+
+            if token_type not in SKIP_TOKENS:
+                yield (token_type, text)
+
+            pos = new_pos
+
+        yield ("EOF", "")
+
+    def _next_token(self, pos: int) -> Tuple[int, str, str]:
+        # find the next token starting at the given position
+        for token_type, pattern in self._token_patterns:
+            match = pattern.match(self.code, pos)
+            if match:
+                return match.end(0), token_type, match.group(0)
+        return None
+
+    @classmethod
+    def from_file(cls, filepath: str, chunk_size: int = 8192) -> "SlangLexer":
+        # create a lexer instance from a file, reading in chunks
+        with open(filepath, "r") as f:
+            return cls(f.read())
diff --git a/crosstl/translator/codegen/directx_codegen.py b/crosstl/translator/codegen/directx_codegen.py
index 7780d31..7cc517f 100644
--- a/crosstl/translator/codegen/directx_codegen.py
+++ b/crosstl/translator/codegen/directx_codegen.py
@@ -43,19 +43,16 @@ class HLSLCodeGen:
         }
 
         self.semantic_map = {
-            # Vertex inputs instance
             "gl_VertexID": "SV_VertexID",
             "gl_InstanceID": "SV_InstanceID",
             "gl_IsFrontFace": "FRONT_FACE",
             "gl_PrimitiveID": "PRIMITIVE_ID",
             "InstanceID": "INSTANCE_ID",
             "VertexID": "VERTEX_ID",
-            # Vertex outputs
             "gl_Position": "SV_POSITION",
             "gl_PointSize": "SV_POINTSIZE",
             "gl_ClipDistance": "SV_ClipDistance",
             "gl_CullDistance": "SV_CullDistance",
-            # Fragment inputs
             "gl_FragColor": "SV_TARGET",
             "gl_FragColor0": "SV_TARGET0",
             "gl_FragColor1": "SV_TARGET1",
@@ -66,10 +63,6 @@ class HLSLCodeGen:
             "gl_FragColor6": "SV_TARGET6",
             "gl_FragColor7": "SV_TARGET7",
             "gl_FragDepth": "SV_DEPTH",
-            "gl_FragDepth0": "SV_DEPTH0",
-            "gl_FragDepth1": "SV_DEPTH1",
-            "gl_FragDepth2": "SV_DEPTH2",
-            "gl_FragDepth3": "SV_DEPTH3",
         }
 
     def generate(self, ast):
@@ -83,7 +76,6 @@ class HLSLCodeGen:
                 code += "}\n"
 
         # Generate global variables
-
         for i, node in enumerate(ast.global_variables):
             if node.vtype in ["sampler2D", "samplerCube"]:
                 code += "// Texture Samplers\n"
@@ -93,6 +85,7 @@ class HLSLCodeGen:
                 code += f"{self.map_type(node.vtype)} {node.name} :register(s{i});\n"
             else:
                 code += f"{self.map_type(node.vtype)} {node.name};\n"
+
         # Generate cbuffers
         if ast.cbuffers:
             code += "// Constant Buffers\n"
@@ -106,7 +99,6 @@ class HLSLCodeGen:
             elif func.qualifier == "fragment":
                 code += "// Fragment Shader\n"
                 code += self.generate_function(func, shader_type="fragment")
-
             elif func.qualifier == "compute":
                 code += "// Compute Shader\n"
                 code += self.generate_function(func, shader_type="compute")
@@ -132,38 +124,32 @@ class HLSLCodeGen:
             f"{self.map_type(p.vtype)} {p.name} {self.map_semantic(p.semantic)}"
             for p in func.params
         )
-        if shader_type == "vertex":
-            code += f"{self.map_type(func.return_type)} VSMain({params}) {self.map_semantic(func.semantic)} {{\n"
-        elif shader_type == "fragment":
-            code += f"{self.map_type(func.return_type)} PSMain({params}) {self.map_semantic(func.semantic)} {{\n"
-        elif shader_type == "compute":
-            code += f"{self.map_type(func.return_type)} CSMain({params}) {self.map_semantic(func.semantic)} {{\n"
+        shader_map = {"vertex": "VSMain", "fragment": "PSMain", "compute": "CSMain"}
+
+        if func.qualifier in shader_map:
+            code += f"// {func.qualifier.capitalize()} Shader\n"
+            code += f"{self.map_type(func.return_type)} {shader_map[func.qualifier]}({params}) {{\n"
         else:
-            code += f"{self.map_type(func.return_type)} {func.name}({params}) {self.map_semantic(func.semantic)} {{\n"
+            code += f"{self.map_type(func.return_type)} {func.name}({params}) {{\n"
 
         for stmt in func.body:
-            code += self.generate_statement(stmt, 1)
-        code += "}\n\n"
-
+            code += self.generate_statement(stmt, indent + 1)
+        code += "  " * indent + "}\n\n"
         return code
 
     def generate_statement(self, stmt, indent=0):
         indent_str = "    " * indent
-        if isinstance(stmt, VariableNode):
-            return f"{indent_str}{self.map_type(stmt.vtype)} {stmt.name};\n"
-        elif isinstance(stmt, AssignmentNode):
-            return f"{indent_str}{self.generate_assignment(stmt)};\n"
-        elif isinstance(stmt, IfNode):
-            return self.generate_if(stmt, indent)
-        elif isinstance(stmt, ForNode):
-            return self.generate_for(stmt, indent)
-        elif isinstance(stmt, ReturnNode):
-            code = ""
-            for i, return_stmt in enumerate(stmt.value):
-                code += f"{self.generate_expression(return_stmt)}"
-                if i < len(stmt.value) - 1:
-                    code += ", "
-            return f"{indent_str}return {code};\n"
+        statement_handlers = {
+            VariableNode: lambda stmt: f"{indent_str}{self.map_type(stmt.vtype)} {stmt.name};\n",
+            AssignmentNode: lambda stmt: f"{indent_str}{self.generate_assignment(stmt)};\n",
+            IfNode: lambda stmt: self.generate_if(stmt, indent),
+            ForNode: lambda stmt: self.generate_for(stmt, indent),
+            ReturnNode: lambda stmt: self.generate_return(stmt, indent),
+        }
+
+        handler = statement_handlers.get(type(stmt))
+        if handler:
+            return handler(stmt)
         else:
             return f"{indent_str}{self.generate_expression(stmt)};\n"
 
@@ -199,15 +185,10 @@ class HLSLCodeGen:
     def generate_for(self, node, indent):
         indent_str = "    " * indent
 
-        init = self.generate_statement(node.init, 0).strip()[
-            :-1
-        ]  # Remove trailing semicolon
-
-        condition = self.generate_statement(node.condition, 0).strip()[
-            :-1
-        ]  # Remove trailing semicolon
-
-        update = self.generate_statement(node.update, 0).strip()[:-1]
+        # Extract and remove the trailing semicolon from init, condition, and update expressions
+        init = self.generate_statement(node.init, 0).strip().rstrip(";")
+        condition = self.generate_statement(node.condition, 0).strip().rstrip(";")
+        update = self.generate_statement(node.update, 0).strip().rstrip(";")
 
         code = f"{indent_str}for ({init}; {condition}; {update}) {{\n"
         for stmt in node.body:
@@ -215,6 +196,15 @@ class HLSLCodeGen:
         code += f"{indent_str}}}\n"
         return code
 
+    def generate_return(self, node, indent):
+        indent_str = "    " * indent
+        code = ""
+        for i, return_stmt in enumerate(node.value):
+            code += f"{self.generate_expression(return_stmt)}"
+            if i < len(node.value) - 1:
+                code += ", "
+        return f"{indent_str}return {code};\n"
+
     def generate_expression(self, expr):
         if isinstance(expr, str):
             return expr
@@ -225,12 +215,10 @@ class HLSLCodeGen:
             left = self.generate_expression(expr.left)
             right = self.generate_expression(expr.right)
             return f"{left} {self.map_operator(expr.op)} {right}"
-
         elif isinstance(expr, AssignmentNode):
             left = self.generate_expression(expr.left)
             right = self.generate_expression(expr.right)
             return f"{left} {self.map_operator(expr.operator)} {right}"
-
         elif isinstance(expr, UnaryOpNode):
             operand = self.generate_expression(expr.operand)
             return f"{self.map_operator(expr.op)}{operand}"
@@ -240,7 +228,6 @@ class HLSLCodeGen:
         elif isinstance(expr, MemberAccessNode):
             obj = self.generate_expression(expr.object)
             return f"{obj}.{expr.member}"
-
         elif isinstance(expr, TernaryOpNode):
             return f"{self.generate_expression(expr.condition)} ? {self.generate_expression(expr.true_expr)} : {self.generate_expression(expr.false_expr)}"
         else:
@@ -285,7 +272,7 @@ class HLSLCodeGen:
         return op_map.get(op, op)
 
     def map_semantic(self, semantic):
-        if semantic is not None:
+        if semantic:
             return f": {self.semantic_map.get(semantic, semantic)}"
         else:
-            return ""
+            return ""  # Handle None by returning an empty string


Test patch:
diff --git a/tests/test_backend/test_metal/test_codegen.py b/tests/test_backend/test_metal/test_codegen.py
index 72e4f74..12a0426 100644
--- a/tests/test_backend/test_metal/test_codegen.py
+++ b/tests/test_backend/test_metal/test_codegen.py
@@ -8,7 +8,7 @@ from crosstl.backend.Metal.MetalCrossGLCodeGen import MetalToCrossGLConverter
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MetalLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_metal/test_lexer.py b/tests/test_backend/test_metal/test_lexer.py
index e745ca7..8ea7b83 100644
--- a/tests/test_backend/test_metal/test_lexer.py
+++ b/tests/test_backend/test_metal/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.Metal.MetalLexer import MetalLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MetalLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_struct_tokenization():
diff --git a/tests/test_backend/test_metal/test_parser.py b/tests/test_backend/test_metal/test_parser.py
index 02286b7..15b2f6f 100644
--- a/tests/test_backend/test_metal/test_parser.py
+++ b/tests/test_backend/test_metal/test_parser.py
@@ -7,7 +7,7 @@ from crosstl.backend.Metal.MetalParser import MetalParser
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MetalLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_mojo/test_codegen.py b/tests/test_backend/test_mojo/test_codegen.py
index 173f817..7ca911b 100644
--- a/tests/test_backend/test_mojo/test_codegen.py
+++ b/tests/test_backend/test_mojo/test_codegen.py
@@ -16,7 +16,7 @@ def generate_code(ast_node):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MojoLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_mojo/test_lexer.py b/tests/test_backend/test_mojo/test_lexer.py
index e23afce..95f7d18 100644
--- a/tests/test_backend/test_mojo/test_lexer.py
+++ b/tests/test_backend/test_mojo/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.Mojo.MojoLexer import MojoLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MojoLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_tokenization():
diff --git a/tests/test_backend/test_mojo/test_parser.py b/tests/test_backend/test_mojo/test_parser.py
index 295781b..4759ac1 100644
--- a/tests/test_backend/test_mojo/test_parser.py
+++ b/tests/test_backend/test_mojo/test_parser.py
@@ -19,7 +19,7 @@ def parse_code(tokens: List):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = MojoLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_parsing():
diff --git a/tests/test_backend/test_opengl/test_codegen.py b/tests/test_backend/test_opengl/test_codegen.py
index bbb83b3..760bfe0 100644
--- a/tests/test_backend/test_opengl/test_codegen.py
+++ b/tests/test_backend/test_opengl/test_codegen.py
@@ -16,7 +16,7 @@ def tokenize_code(code: str) -> List:
 
     """
     lexer = GLSLLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(Tokens: List, shader_type="vertex") -> List:
diff --git a/tests/test_backend/test_opengl/test_lexer.py b/tests/test_backend/test_opengl/test_lexer.py
index a112929..5226520 100644
--- a/tests/test_backend/test_opengl/test_lexer.py
+++ b/tests/test_backend/test_opengl/test_lexer.py
@@ -14,7 +14,7 @@ def tokenize_code(code: str) -> List:
 
     """
     lexer = GLSLLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_input_output_tokenization():
diff --git a/tests/test_backend/test_opengl/test_parser.py b/tests/test_backend/test_opengl/test_parser.py
index 79a107b..5951406 100644
--- a/tests/test_backend/test_opengl/test_parser.py
+++ b/tests/test_backend/test_opengl/test_parser.py
@@ -15,7 +15,7 @@ def tokenize_code(code: str) -> List:
 
     """
     lexer = GLSLLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(Tokens: List, shader_type="vertex") -> List:
diff --git a/tests/test_backend/test_slang/test_codegen.py b/tests/test_backend/test_slang/test_codegen.py
index 6a00430..cb5387f 100644
--- a/tests/test_backend/test_slang/test_codegen.py
+++ b/tests/test_backend/test_slang/test_codegen.py
@@ -19,7 +19,7 @@ def generate_code(ast_node):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = SlangLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_slang/test_lexer.py b/tests/test_backend/test_slang/test_lexer.py
index a9bbd98..1ef8bd2 100644
--- a/tests/test_backend/test_slang/test_lexer.py
+++ b/tests/test_backend/test_slang/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.slang.SlangLexer import SlangLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = SlangLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_struct_tokenization():
diff --git a/tests/test_backend/test_slang/test_parser.py b/tests/test_backend/test_slang/test_parser.py
index 69fd232..742f597 100644
--- a/tests/test_backend/test_slang/test_parser.py
+++ b/tests/test_backend/test_slang/test_parser.py
@@ -19,7 +19,7 @@ def parse_code(tokens: List):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = SlangLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_struct_parsing():
diff --git a/tests/test_backend/test_vulkan/test_codegen.py b/tests/test_backend/test_vulkan/test_codegen.py
index 6e2c91c..71a274d 100644
--- a/tests/test_backend/test_vulkan/test_codegen.py
+++ b/tests/test_backend/test_vulkan/test_codegen.py
@@ -16,7 +16,7 @@ def generate_code(ast_node):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = VulkanLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def parse_code(tokens: List):
diff --git a/tests/test_backend/test_vulkan/test_lexer.py b/tests/test_backend/test_vulkan/test_lexer.py
index 9fabf09..e68db35 100644
--- a/tests/test_backend/test_vulkan/test_lexer.py
+++ b/tests/test_backend/test_vulkan/test_lexer.py
@@ -6,7 +6,7 @@ from crosstl.backend.Vulkan.VulkanLexer import VulkanLexer
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = VulkanLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_tokenization():
diff --git a/tests/test_backend/test_vulkan/test_parser.py b/tests/test_backend/test_vulkan/test_parser.py
index d1af630..ec7789a 100644
--- a/tests/test_backend/test_vulkan/test_parser.py
+++ b/tests/test_backend/test_vulkan/test_parser.py
@@ -19,7 +19,7 @@ def parse_code(tokens: List):
 def tokenize_code(code: str) -> List:
     """Helper function to tokenize code."""
     lexer = VulkanLexer(code)
-    return lexer.tokens
+    return lexer.tokenize()
 
 
 def test_mod_parsing():