Repository: CrossGL/crosstl

Issue:
#include
<!-- BOT_STATE: {"assignee": "SumitBahl02", "assigned_at": "2024-12-26T13:56:49.951114+00:00", "reminder_sent": false, "unassigned": false} -->

Discussion:
@SumitBahl02: @CrossGL-issue-bot assign me
please assign #196 to me.@NripeshN: > @CrossGL-issue-bot assign me please assign #196 to me.

I believe @plon-Susk7 already created a PR for this issue, feel free to collaborate with them or create a work on a different issue.

Solution patch:
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index c75ed284..a9342056 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,48 +1,73 @@
-### PR Description
-<!-- Provide a brief summary of the changes you have made. Explain the purpose and motivation behind these changes. -->
+<!--
+Hey there! Thanks for opening a PR.
+
+This file is a template for pull requests. 
+The comments (in <!-- -\->) are only visible in the raw markdown,
+
+so they won't show up in the final PR description.
+
+Fill out each section briefly. Keep it simple.
+-->
+
+### Summary
+
+<!--
+Short and to the point: What does this PR do and why?
+Example: "Adds Perlin noise to the fragment shader to generate a swirl effect."
+-->
 
 ### Related Issue
-<!-- Link to the related issue(s) that this PR addresses. Example: #123 -->
-
-### shader Sample
-<!-- Provide a shader sample or snippet on which you have tested your changes. like
-
-```crossgl
-shader PerlinNoise {
-    vertex {
-        input vec3 position;
-        output vec2 vUV;
-
-        void main() {
-            vUV = position.xy * 10.0;
-            gl_Position = vec4(position, 1.0);
-        }
-    }
-
-    // Perlin Noise Function
-    float perlinNoise(vec2 p) {
-        return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453);
-    }
-
-    // Fragment Shader
-    fragment {
-        input vec2 vUV;
-        output vec4 fragColor;
-
-        void main() {
-            float noise = perlinNoise(vUV);
-            float height = noise * 10.0;
-            vec3 color = vec3(height / 10.0, 1.0 - height / 10.0, 0.0);
-            fragColor = vec4(color, 1.0);
-        }
-    }
+
+<!--
+If this PR addresses an open issue, use a phrase like:
+closes #123
+fixes #123
+resolves #123
+
+That way, GitHub links the issue to this PR and closes it automatically on merge.
+If it’s not fixing a specific issue, say so.
+-->
+
+### Changes
+
+<!--
+List the core changes.
+Example:
+- Implemented `perlinNoise()` in the fragment shader.
+- Tweaked the color output to highlight the noise value.
+-->
+
+### Shader Sample (Optional)
+
+<!--
+If you're touching shader code, provide a short snippet
+to show how you've tested or integrated these changes.
+
+Example:
+\`\`\`crossgl
+shader ExampleShader {
+    // ...
 }
-```-->
+\`\`\`
+-->
+
+### Testing
 
+<!--
+Tell us how you tested your changes.
+- Did you run existing tests?
+- Add any new tests?
+- Manually tested in a local environment?
+-->
 
 ### Checklist
-- [ ] Have you added the necessary tests?
-- [ ] Only modified the files mentioned in the related issue(s)?
-- [ ] Are all tests passing?
 
+<!--
+Check off each item as you go.
+Feel free to add or remove items that fit your repo's workflow.
+-->
 
+- [ ] Tests cover new or changed code (or reason why not)
+- [ ] Linked the issue with a closing keyword (if applicable)
+- [ ] Only touched files related to the issue
+- [ ] Everything builds and runs locally
diff --git a/crosstl/backend/DirectX/DirectxAst.py b/crosstl/backend/DirectX/DirectxAst.py
index ac88d52b..32696638 100644
--- a/crosstl/backend/DirectX/DirectxAst.py
+++ b/crosstl/backend/DirectX/DirectxAst.py
@@ -159,3 +159,14 @@ def __repr__(self):
 
     def __str__(self):
         return f"({self.op}{self.operand})"
+
+
+class IncludeNode(ASTNode):
+    def __init__(self, path):
+        self.path = path
+
+    def __repr__(self):
+        return f"IncludeNode(path={self.path})"
+
+    def __str__(self):
+        return f"#include {self.path}"
diff --git a/crosstl/backend/DirectX/DirectxCrossGLCodeGen.py b/crosstl/backend/DirectX/DirectxCrossGLCodeGen.py
index 04a4b632..103f98cc 100644
--- a/crosstl/backend/DirectX/DirectxCrossGLCodeGen.py
+++ b/crosstl/backend/DirectX/DirectxCrossGLCodeGen.py
@@ -70,6 +70,8 @@ def generate(self, ast):
                 for member in node.members:
                     code += f"        {self.map_type(member.vtype)} {member.name} {self.map_semantic(member.semantic)};\n"
                 code += "    }\n"
+            elif isinstance(node, IncludeNode):
+                code += f"    #include {node.path}\n"
         # Generate global variables
         for node in ast.global_variables:
             code += f"    {self.map_type(node.vtype)} {node.name};\n"
diff --git a/crosstl/backend/DirectX/DirectxLexer.py b/crosstl/backend/DirectX/DirectxLexer.py
index a2c6084f..d7e2e254 100644
--- a/crosstl/backend/DirectX/DirectxLexer.py
+++ b/crosstl/backend/DirectX/DirectxLexer.py
@@ -3,6 +3,7 @@
 TOKENS = [
     ("COMMENT_SINGLE", r"//.*"),
     ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+    ("INCLUDE", r"\#include\b"),
     ("STRUCT", r"\bstruct\b"),
     ("CBUFFER", r"\bcbuffer\b"),
     ("TEXTURE2D", r"\bTexture2D\b"),
@@ -50,8 +51,8 @@
     ("ASSIGN_OR", r"\|="),
     ("ASSIGN_AND", r"\&="),
     ("BITWISE_XOR", r"\^"),
-    ("AND", r"&&"),
-    ("OR", r"\|\|"),
+    ("LOGICAL_AND", r"&&"),
+    ("LOGICAL_OR", r"\|\|"),
     ("BITWISE_OR", r"\|"),
     ("DOT", r"\."),
     ("MULTIPLY", r"\*"),
@@ -60,6 +61,7 @@
     ("MINUS", r"-"),
     ("EQUALS", r"="),
     ("WHITESPACE", r"\s+"),
+    ("STRING", r"\"[^\"]*\""),  # Added for string literals
 ]
 
 KEYWORDS = {
@@ -95,8 +97,10 @@ def tokenize(self):
         pos = 0
         while pos < len(self.code):
             match = None
+
             for token_type, pattern in TOKENS:
                 regex = re.compile(pattern)
+
                 match = regex.match(self.code, pos)
                 if match:
                     text = match.group(0)
diff --git a/crosstl/backend/DirectX/DirectxParser.py b/crosstl/backend/DirectX/DirectxParser.py
index c3cd409d..011c9fcc 100644
--- a/crosstl/backend/DirectX/DirectxParser.py
+++ b/crosstl/backend/DirectX/DirectxParser.py
@@ -14,6 +14,7 @@
     UnaryOpNode,
     VariableNode,
     VectorConstructorNode,
+    IncludeNode,
 )
 from .DirectxLexer import HLSLLexer
 
@@ -66,11 +67,20 @@ def parse_shader(self):
                     functions.append(self.parse_function())
                 else:
                     global_variables.append(self.parse_global_variable())
+            elif self.current_token[0] == "INCLUDE":
+                structs.append(self.parse_include())
+
             else:
                 self.eat(self.current_token[0])  # Skip unknown tokens
 
         return ShaderNode(structs, functions, global_variables, cbuffers)
 
+    def parse_include(self):
+        self.eat("INCLUDE")
+        path = self.current_token[1]
+        self.eat("STRING")
+        return IncludeNode(path)
+
     def is_function(self):
         current_pos = self.pos
         while self.tokens[current_pos][0] != "EOF":
@@ -428,18 +438,18 @@ def parse_assignment(self):
 
     def parse_logical_or(self):
         left = self.parse_logical_and()
-        while self.current_token[0] == "OR":
+        while self.current_token[0] == "LOGICAL_OR":
             op = self.current_token[1]
-            self.eat("OR")
+            self.eat("LOGICAL_OR")
             right = self.parse_logical_and()
             left = BinaryOpNode(left, op, right)
         return left
 
     def parse_logical_and(self):
         left = self.parse_equality()
-        while self.current_token[0] == "AND":
+        while self.current_token[0] == "LOGICAL_AND":
             op = self.current_token[1]
-            self.eat("AND")
+            self.eat("LOGICAL_AND")
             right = self.parse_equality()
             left = BinaryOpNode(left, op, right)
         return left
diff --git a/crosstl/translator/codegen/directx_codegen.py b/crosstl/translator/codegen/directx_codegen.py
index 7f390a29..7780d31a 100644
--- a/crosstl/translator/codegen/directx_codegen.py
+++ b/crosstl/translator/codegen/directx_codegen.py
@@ -258,6 +258,7 @@ def map_operator(self, op):
             "MULTIPLY": "*",
             "DIVIDE": "/",
             "BITWISE_XOR": "^",
+            "BITWISE_OR": "|",
             "BITWISE_AND": "&",
             "LESS_THAN": "<",
             "GREATER_THAN": ">",
diff --git a/crosstl/translator/codegen/metal_codegen.py b/crosstl/translator/codegen/metal_codegen.py
index 11b59495..fe25fadf 100644
--- a/crosstl/translator/codegen/metal_codegen.py
+++ b/crosstl/translator/codegen/metal_codegen.py
@@ -343,6 +343,7 @@ def map_operator(self, op):
             "MULTIPLY": "*",
             "DIVIDE": "/",
             "BITWISE_XOR": "^",
+            "BITWISE_OR": "|",
             "BITWISE_AND": "&",
             "LESS_THAN": "<",
             "GREATER_THAN": ">",
@@ -364,7 +365,6 @@ def map_operator(self, op):
             "ASSIGN_SHIFT_RIGHT": ">>=",
             "ASSIGN_AND": "&=",
             "LOGICAL_AND": "&&",
-            "ASSIGN_XOR": "^=",
             "BITWISE_SHIFT_RIGHT": ">>",
             "BITWISE_SHIFT_LEFT": "<<",
         }
diff --git a/crosstl/translator/lexer.py b/crosstl/translator/lexer.py
index e6540499..ac89f534 100644
--- a/crosstl/translator/lexer.py
+++ b/crosstl/translator/lexer.py
@@ -1,81 +1,82 @@
 import re
+from collections import OrderedDict
 
-TOKENS = [
-    ("COMMENT_SINGLE", r"//.*"),
-    ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
-    ("SHADER", r"\bshader\b"),
-    ("VOID", r"\bvoid\b"),
-    ("STRUCT", r"\bstruct\b"),
-    ("CBUFFER", r"\bcbuffer\b"),
-    ("AT", r"\@"),
-    ("SAMPLER2D", r"\bsampler2D\b"),
-    ("SAMPLERCUBE", r"\bsamplerCube\b"),
-    ("VECTOR", r"\bvec[2-4]\b"),
-    ("MATRIX", r"\bmat[2-4]\b"),
-    ("BOOL", r"\bbool\b"),
-    ("VERTEX", r"\bvertex\b"),
-    ("FRAGMENT", r"\bfragment\b"),
-    ("FLOAT_NUMBER", r"\d*\.\d+|\d+\.\d*"),
-    ("FLOAT", r"\bfloat\b"),
-    ("INT", r"\bint\b"),
-    ("UINT", r"\buint\b"),
-    ("DOUBLE", r"\bdouble\b"),
-    ("SAMPLER", r"\bsampler\b"),
-    ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z_0-9]*"),
-    ("ASSIGN_SHIFT_RIGHT", r">>="),
-    ("ASSIGN_SHIFT_LEFT", r"<<="),
-    ("NUMBER", r"\d+(\.\d+)?"),
-    ("LBRACE", r"\{"),
-    ("RBRACE", r"\}"),
-    ("LPAREN", r"\("),
-    ("RPAREN", r"\)"),
-    ("SEMICOLON", r";"),
-    ("COMMA", r","),
-    ("ASSIGN_ADD", r"\+="),
-    ("ASSIGN_SUB", r"-="),
-    ("ASSIGN_MUL", r"\*="),
-    ("ASSIGN_DIV", r"/="),
-    ("WHITESPACE", r"\s+"),
-    ("IF", r"\bif\b"),
-    ("ELSE", r"\belse\b"),
-    ("FOR", r"\bfor\b"),
-    ("RETURN", r"\breturn\b"),
-    ("BITWISE_SHIFT_LEFT", r"<<"),
-    ("BITWISE_SHIFT_RIGHT", r">>"),
-    ("LESS_EQUAL", r"<="),
-    ("GREATER_EQUAL", r">="),
-    ("GREATER_THAN", r">"),
-    ("LESS_THAN", r"<"),
-    ("INCREMENT", r"\+\+"),
-    ("DECREMENT", r"--"),
-    ("EQUAL", r"=="),
-    ("NOT_EQUAL", r"!="),
-    ("ASSIGN_AND", r"&="),
-    ("ASSIGN_OR", r"\|="),
-    ("ASSIGN_XOR", r"\^="),
-    ("LOGICAL_AND", r"&&"),
-    ("LOGICAL_OR", r"\|\|"),
-    ("NOT", r"!"),
-    ("ASSIGN_MOD", r"%="),
-    ("MOD", r"%"),
-    ("INCREMENT", r"\+\+"),
-    ("DECREMENT", r"\-\-"),
-    ("PLUS", r"\+"),
-    ("MINUS", r"-"),
-    ("MULTIPLY", r"\*"),
-    ("DIVIDE", r"/"),
-    ("DOT", r"\."),
-    ("EQUALS", r"="),
-    ("QUESTION", r"\?"),
-    ("COLON", r":"),
-    ("CONST", r"\bconst\b"),
-    ("BITWISE_AND", r"&"),
-    ("BITWISE_OR", r"\|"),
-    ("BITWISE_XOR", r"\^"),
-    ("BITWISE_NOT", r"~"),
-    ("shift_left", r"<<"),
-    ("shift_right", r">>"),
-]
+TOKENS = OrderedDict(
+    [
+        ("COMMENT_SINGLE", r"//.*"),
+        ("COMMENT_MULTI", r"/\*[\s\S]*?\*/"),
+        ("SHADER", r"\bshader\b"),
+        ("VOID", r"\bvoid\b"),
+        ("STRUCT", r"\bstruct\b"),
+        ("CBUFFER", r"\bcbuffer\b"),
+        ("AT", r"\@"),
+        ("SAMPLER2D", r"\bsampler2D\b"),
+        ("SAMPLERCUBE", r"\bsamplerCube\b"),
+        ("VECTOR", r"\bvec[2-4]\b"),
+        ("MATRIX", r"\bmat[2-4]\b"),
+        ("BOOL", r"\bbool\b"),
+        ("VERTEX", r"\bvertex\b"),
+        ("FRAGMENT", r"\bfragment\b"),
+        ("FLOAT_NUMBER", r"\d*\.\d+|\d+\.\d*"),
+        ("FLOAT", r"\bfloat\b"),
+        ("INT", r"\bint\b"),
+        ("UINT", r"\buint\b"),
+        ("DOUBLE", r"\bdouble\b"),
+        ("SAMPLER", r"\bsampler\b"),
+        ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z_0-9]*"),
+        ("ASSIGN_SHIFT_RIGHT", r">>="),
+        ("ASSIGN_SHIFT_LEFT", r"<<="),
+        ("NUMBER", r"\d+(\.\d+)?"),
+        ("LBRACE", r"\{"),
+        ("RBRACE", r"\}"),
+        ("LPAREN", r"\("),
+        ("RPAREN", r"\)"),
+        ("SEMICOLON", r";"),
+        ("COMMA", r","),
+        ("ASSIGN_ADD", r"\+="),
+        ("ASSIGN_SUB", r"-="),
+        ("ASSIGN_MUL", r"\*="),
+        ("ASSIGN_DIV", r"/="),
+        ("WHITESPACE", r"\s+"),
+        ("IF", r"\bif\b"),
+        ("ELSE", r"\belse\b"),
+        ("FOR", r"\bfor\b"),
+        ("RETURN", r"\breturn\b"),
+        ("BITWISE_SHIFT_LEFT", r"<<"),
+        ("BITWISE_SHIFT_RIGHT", r">>"),
+        ("LESS_EQUAL", r"<="),
+        ("GREATER_EQUAL", r">="),
+        ("GREATER_THAN", r">"),
+        ("LESS_THAN", r"<"),
+        ("INCREMENT", r"\+\+"),
+        ("DECREMENT", r"--"),
+        ("EQUAL", r"=="),
+        ("NOT_EQUAL", r"!="),
+        ("ASSIGN_AND", r"&="),
+        ("ASSIGN_OR", r"\|="),
+        ("ASSIGN_XOR", r"\^="),
+        ("LOGICAL_AND", r"&&"),
+        ("LOGICAL_OR", r"\|\|"),
+        ("NOT", r"!"),
+        ("ASSIGN_MOD", r"%="),
+        ("MOD", r"%"),
+        ("INCREMENT", r"\+\+"),
+        ("DECREMENT", r"\-\-"),
+        ("PLUS", r"\+"),
+        ("MINUS", r"-"),
+        ("MULTIPLY", r"\*"),
+        ("DIVIDE", r"/"),
+        ("DOT", r"\."),
+        ("EQUALS", r"="),
+        ("QUESTION", r"\?"),
+        ("COLON", r":"),
+        ("CONST", r"\bconst\b"),
+        ("BITWISE_AND", r"&"),
+        ("BITWISE_OR", r"\|"),
+        ("BITWISE_XOR", r"\^"),
+        ("BITWISE_NOT", r"~"),
+    ]
+)
 
 KEYWORDS = {
     "shader": "SHADER",
@@ -91,46 +92,61 @@
 
 
 class Lexer:
-    """A simple lexer for the shader language
+    """A simple lexer for the shader language with optimizations
 
     This lexer tokenizes the input code into a list of tokens.
+    Includes optimizations:
+    - Token caching for frequently used tokens
+    - Combined regex patterns for similar tokens
+    - Precompiled regex patterns
 
     Attributes:
         code (str): The input code to tokenize
         tokens (list): A list of tokens generated from the input code
-
+        token_cache (dict): Cache for frequently used tokens
     """
 
     def __init__(self, code):
         self.code = code
         self.tokens = []
+        self.token_cache = {}
+        self.regex_cache = self._compile_patterns()
         self.tokenize()
 
-    def tokenize(self):
-        pos = 0
-        while pos < len(self.code):
-            match = None
-            for token_type, pattern in TOKENS:
-                regex = re.compile(pattern)
-                match = regex.match(self.code, pos)
-                if match:
-                    text = match.group(0)
-                    if token_type == "IDENTIFIER" and text in KEYWORDS:
-                        token_type = KEYWORDS[text]
-                    if token_type != "WHITESPACE":  # Ignore whitespace tokens
+    def _compile_patterns(self):
+        """Compile a single regex with named groups for all tokens."""
+        combined_pattern = "|".join(
+            f"(?P<{name}>{pattern})" for name, pattern in TOKENS.items()
+        )
+        return re.compile(combined_pattern)
 
-                        token = (token_type, text)
+    def _get_cached_token(self, text, token_type):
+        """Return a cached token tuple if possible."""
+        cache_key = (text, token_type)
+        if cache_key not in self.token_cache:
+            self.token_cache[cache_key] = (token_type, text)
+        return self.token_cache[cache_key]
 
-                        self.tokens.append(token)
-                    pos = match.end(0)
-                    break
-            if not match:
-                unmatched_char = self.code[pos]
-                highlighted_code = (
-                    self.code[:pos] + "[" + self.code[pos] + "]" + self.code[pos + 1 :]
+    def tokenize(self):
+        pos = 0
+        length = len(self.code)
+        while pos < length:
+            match = self.regex_cache.match(self.code, pos)
+            if match:
+                token_type = match.lastgroup
+                text = match.group(token_type)
+                if token_type == "IDENTIFIER" and text in KEYWORDS:
+                    token_type = KEYWORDS[text]
+                if token_type != "WHITESPACE":
+                    token = self._get_cached_token(text, token_type)
+                    self.tokens.append(token)
+                pos = match.end(0)
+            else:
+                bad_char = self.code[pos]
+                highlighted = (
+                    self.code[:pos] + "[" + bad_char + "]" + self.code[pos + 1 :]
                 )
                 raise SyntaxError(
-                    f"Illegal character '{unmatched_char}' at position {pos}\n{highlighted_code}"
+                    f"Illegal character '{bad_char}' at position {pos}\n{highlighted}"
                 )
-
-        self.tokens.append(("EOF", None))  # End of file token
+        self.tokens.append(self._get_cached_token(None, "EOF"))
diff --git a/crosstl/translator/parser.py b/crosstl/translator/parser.py
index eeca778c..d9cbccfa 100644
--- a/crosstl/translator/parser.py
+++ b/crosstl/translator/parser.py
@@ -16,6 +16,7 @@
 )
 
 from .lexer import Lexer
+import warnings
 
 
 class Parser:
@@ -112,6 +113,9 @@ def parse_shader(self):
                 else:
                     global_variables.append(self.parse_global_variable())
             else:
+                warnings.warn(
+                    f"Skipping unexpected token {self.current_token[0]}", SyntaxWarning
+                )
                 self.eat(self.current_token[0])  # Skip unknown tokens
 
         return ShaderNode(structs, functions, global_variables, cbuffers)


Test patch:
diff --git a/tests/test_backend/test_directx/test_codegen.py b/tests/test_backend/test_directx/test_codegen.py
index 7d746208..4c0ccfb4 100644
--- a/tests/test_backend/test_directx/test_codegen.py
+++ b/tests/test_backend/test_directx/test_codegen.py
@@ -498,5 +498,46 @@ def test_bitwise_ops_codgen():
         pytest.fail("bitwise_op parsing or codegen not implemented.")
 
 
+def test_include_codegen():
+    code = """
+    #include "common.hlsl"
+    struct VSInput {
+    float4 position : POSITION;
+    float4 color : TEXCOORD0;
+    };
+
+    struct VSOutput {
+        float4 out_position : TEXCOORD0;
+    };
+
+    VSOutput VSMain(VSInput input) {
+        VSOutput output;
+        output.out_position =  input.position;
+        return output;
+    }
+
+    struct PSInput {
+        float4 in_position : TEXCOORD0;
+    };
+
+    struct PSOutput {
+        float4 out_color : SV_TARGET0;
+    };
+
+    PSOutput PSMain(PSInput input) {
+        PSOutput output;
+        output.out_color =  input.in_position;
+        return output;
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        ast = parse_code(tokens)
+        generated_code = generate_code(ast)
+        print(generated_code)
+    except SyntaxError:
+        pytest.fail("Include statement failed to parse or generate code.")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_directx/test_lexer.py b/tests/test_backend/test_directx/test_lexer.py
index 0b12c360..9318f12c 100644
--- a/tests/test_backend/test_directx/test_lexer.py
+++ b/tests/test_backend/test_directx/test_lexer.py
@@ -182,5 +182,27 @@ def test_bitwise_or_tokenization():
         pytest.fail("bitwise_op tokenization is not implemented.")
 
 
+def test_logical_or_tokenization():
+    code = """
+        bool val_0 = true;
+        bool val_1 = val_0 || false;
+    """
+    try:
+        tokenize_code(code)
+    except SyntaxError:
+        pytest.fail("logical_or tokenization is not implemented.")
+
+
+def test_logical_and_tokenization():
+    code = """
+        bool val_0 = true;
+        bool val_1 = val_0 && false;
+    """
+    try:
+        tokenize_code(code)
+    except SyntaxError:
+        pytest.fail("logical_and tokenization is not implemented.")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_directx/test_parser.py b/tests/test_backend/test_directx/test_parser.py
index dd73e9df..b85b03d5 100644
--- a/tests/test_backend/test_directx/test_parser.py
+++ b/tests/test_backend/test_directx/test_parser.py
@@ -257,5 +257,55 @@ def test_bitwise_ops_parsing():
         pytest.fail("bitwise_op parsing not implemented.")
 
 
+def test_logical_or_ops_parsing():
+    code = """
+            PSOutput PSMain(PSInput input) {
+                PSOutput output;
+                output.out_color = float4(0.0, 0.0, 0.0, 1.0);
+                // Test case for logical OR
+                bool condition1 = true; // First condition
+                bool condition2 = false; // Second condition
+                if (condition1 || condition2) {
+                    // If one of the condition is true
+                    output.out_color = float4(1.0, 0.0, 0.0, 1.0); // Set color to red
+                } else {
+                    // If both of the conditions are false
+                    output.out_color = float4(0.0, 1.0, 0.0, 1.0); // Set color to green
+                }
+                return output;
+            }
+        """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("logical_or_ops not implemented.")
+
+
+def test_logical_and_ops_parsing():
+    code = """
+            PSOutput PSMain(PSInput input) {
+                PSOutput output;
+                output.out_color = float4(0.0, 0.0, 0.0, 1.0);
+                // Test case for logical AND
+                bool condition1 = true; // First condition
+                bool condition2 = false; // Second condition
+                if (condition1 && condition2) {
+                    // both the condition is true
+                    output.out_color = float4(1.0, 0.0, 0.0, 1.0); // Set color to red
+                } else {
+                    // any one of the condition is false
+                    output.out_color = float4(0.0, 1.0, 0.0, 1.0); // Set color to green
+                }
+                return output;
+            }
+        """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("logical_and_ops not implemented.")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_translator/test_codegen/test_directx_codegen.py b/tests/test_translator/test_codegen/test_directx_codegen.py
index c55d5929..e7f627c2 100644
--- a/tests/test_translator/test_codegen/test_directx_codegen.py
+++ b/tests/test_translator/test_codegen/test_directx_codegen.py
@@ -683,5 +683,42 @@ def test_shift_operators():
         pytest.fail("Shift right assignment operator codegen not implemented.")
 
 
+def test_bitwise_or_operator():
+    code = """
+    shader main {
+    struct VSInput {
+        vec2 texCoord @ TEXCOORD0;
+    };
+    struct VSOutput {
+        vec4 color @ COLOR;
+    };
+    sampler2D iChannel0;
+    vertex {
+        VSOutput main(VSInput input) {
+            VSOutput output;
+            // Use bitwise OR on texture coordinates (for testing purposes)
+            output.color = vec4(float(int(input.texCoord.x * 100.0) | 15), 
+                                float(int(input.texCoord.y * 100.0) | 15), 
+                                0.0, 1.0);
+            return output;
+        }
+    }
+    fragment {
+        vec4 main(VSOutput input) @ gl_FragColor {
+            // Simple fragment shader to display the result of the AND operation
+            return vec4(input.color.rgb, 1.0);
+        }
+    }
+}
+    """
+    try:
+        tokens = tokenize_code(code)
+        ast = parse_code(tokens)
+        generated_code = generate_code(ast)
+        print(generated_code)
+    except SyntaxError:
+        pytest.fail("Bitwise OR codegen not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_translator/test_codegen/test_metal_codegen.py b/tests/test_translator/test_codegen/test_metal_codegen.py
index 8ed86810..6d947cfb 100644
--- a/tests/test_translator/test_codegen/test_metal_codegen.py
+++ b/tests/test_translator/test_codegen/test_metal_codegen.py
@@ -366,5 +366,42 @@ def test_bitwise_operators():
         pytest.fail("Bitwise Shift codegen not implemented.")
 
 
+def test_bitwise_or_operator():
+    code = """
+    shader main {
+    struct VSInput {
+        vec2 texCoord @ TEXCOORD0;
+    };
+    struct VSOutput {
+        vec4 color @ COLOR;
+    };
+    sampler2D iChannel0;
+    vertex {
+        VSOutput main(VSInput input) {
+            VSOutput output;
+            // Use bitwise AND on texture coordinates (for testing purposes)
+            output.color = vec4(float(int(input.texCoord.x * 100.0) | 15), 
+                                float(int(input.texCoord.y * 100.0) | 15), 
+                                0.0, 1.0);
+            return output;
+        }
+    }
+    fragment {
+        vec4 main(VSOutput input) @ gl_FragColor {
+            // Simple fragment shader to display the result of the AND operation
+            return vec4(input.color.rgb, 1.0);
+        }
+    }
+}
+    """
+    try:
+        tokens = tokenize_code(code)
+        ast = parse_code(tokens)
+        generated_code = generate_code(ast)
+        print(generated_code)
+    except SyntaxError:
+        pytest.fail("Bitwise OR codegen not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_translator/test_lexer.py b/tests/test_translator/test_lexer.py
index abff042f..35c778a1 100644
--- a/tests/test_translator/test_lexer.py
+++ b/tests/test_translator/test_lexer.py
@@ -1,6 +1,6 @@
-from crosstl.translator.lexer import Lexer
 import pytest
 from typing import List
+from crosstl.translator.lexer import Lexer
 
 
 def tokenize_code(code: str) -> List:
@@ -21,7 +21,8 @@ def test_struct_tokenization():
 
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "STRUCT" for t in tokens), "Missing 'STRUCT' token"
     except SyntaxError:
         pytest.fail("Struct tokenization not implemented.")
 
@@ -35,7 +36,9 @@ def test_if_statement_tokenization():
     }
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "IF" for t in tokens), "Missing 'IF' token"
+        assert any(t[0] == "ELSE" for t in tokens), "Missing 'ELSE' token"
     except SyntaxError:
         pytest.fail("if tokenization not implemented.")
 
@@ -47,7 +50,8 @@ def test_for_statement_tokenization():
     }
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "FOR" for t in tokens), "Missing 'FOR' token"
     except SyntaxError:
         pytest.fail("for tokenization not implemented.")
 
@@ -61,7 +65,8 @@ def test_else_statement_tokenization():
     }
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "ELSE" for t in tokens), "Missing 'ELSE' token"
     except SyntaxError:
         pytest.fail("else tokenization not implemented.")
 
@@ -82,9 +87,10 @@ def test_else_if_statement_tokenization():
     }
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert tokens, "No tokens generated"
     except SyntaxError:
-        pytest.fail("else if  tokenization not implemented.")
+        pytest.fail("else if tokenization not implemented.")
 
 
 def test_function_call_tokenization():
@@ -98,11 +104,9 @@ def test_function_call_tokenization():
     
     fragment {
         vec4 main(VSOutput input) @ gl_FragColor {
-            // Sample brightness and calculate bloom
             float brightness = texture(iChannel0, input.color.xy).r;
             float bloom = max(0.0, brightness - 0.5);
             bloom = perlinNoise(input.color.xy);
-            // Apply bloom to the texture color
             vec3 texColor = texture(iChannel0, input.color.xy).rgb;
             vec3 colorWithBloom = texColor + vec3(bloom);
 
@@ -112,7 +116,9 @@ def test_function_call_tokenization():
 }
 """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "SHADER" for t in tokens), "Missing 'SHADER' token"
+        assert any(t[0] == "FRAGMENT" for t in tokens), "Missing 'FRAGMENT' token"
     except SyntaxError:
         pytest.fail("Function call tokenization not implemented.")
 
@@ -122,15 +128,19 @@ def test_bitwise_operator_tokenization():
     int a = 60; // 60 = 0011 1100
     int b = 13; // 13 = 0000 1101
     int c = 0;
-    c = a & b; // 12 = 0000 1100
-    c = a | b; // 61 = 0011 1101
-    c = a ^ b; // 49 = 0011 0001
-    c = ~a; // -61 = 1100 0011
-    c = a << 2; // 240 = 1111 0000
-    c = a >> 2; // 15 = 0000 1111
-      """
+    c = a & b; 
+    c = a | b; 
+    c = a ^ b; 
+    c = ~a; 
+    c = a << 2; 
+    c = a >> 2; 
+    """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "BITWISE_AND" for t in tokens), "Missing '&' token"
+        assert any(t[0] == "BITWISE_OR" for t in tokens), "Missing '|' token"
+        assert any(t[0] == "BITWISE_XOR" for t in tokens), "Missing '^' token"
+        assert any(t[0] == "BITWISE_NOT" for t in tokens), "Missing '~' token"
     except SyntaxError:
         pytest.fail("Bitwise operator tokenization not implemented.")
 
@@ -144,7 +154,12 @@ def test_data_types_tokenization():
     bool e;
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "INT" for t in tokens), "Missing 'INT' token"
+        assert any(t[0] == "UINT" for t in tokens), "Missing 'UINT' token"
+        assert any(t[0] == "FLOAT" for t in tokens), "Missing 'FLOAT' token"
+        assert any(t[0] == "DOUBLE" for t in tokens), "Missing 'DOUBLE' token"
+        assert any(t[0] == "BOOL" for t in tokens), "Missing 'BOOL' token"
     except SyntaxError:
         pytest.fail("Data types tokenization not implemented.")
 
@@ -153,13 +168,18 @@ def test_operators_tokenization():
     code = """
     int a;
     a = 2 + 1;
-    a = a -  2;
+    a = a - 2;
     a = a / 1;
     a = a * 2;
     a = a % 2;
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "PLUS" for t in tokens), "Missing '+' token"
+        assert any(t[0] == "MINUS" for t in tokens), "Missing '-' token"
+        assert any(t[0] == "DIVIDE" for t in tokens), "Missing '/' token"
+        assert any(t[0] == "MULTIPLY" for t in tokens), "Missing '*' token"
+        assert any(t[0] == "MOD" for t in tokens), "Missing '%' token"
     except SyntaxError:
         pytest.fail("Operators tokenization not implemented.")
 
@@ -173,7 +193,9 @@ def test_logical_operators_tokenization():
     }
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "LOGICAL_OR" for t in tokens), "Missing '||' token"
+        assert any(t[0] == "LOGICAL_AND" for t in tokens), "Missing '&&' token"
     except SyntaxError:
         pytest.fail("Logical Operators tokenization not implemented.")
 
@@ -182,10 +204,11 @@ def test_assignment_shift_operators():
     code = """
     a >>= 1;
     b <<= 1;
-        """
-
+    """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "ASSIGN_SHIFT_RIGHT" for t in tokens), "Missing '>>=' token"
+        assert any(t[0] == "ASSIGN_SHIFT_LEFT" for t in tokens), "Missing '<<=' token"
     except SyntaxError:
         pytest.fail("Shift operators tokenization failed.")
 
@@ -203,7 +226,15 @@ def test_assignment_operators_tokenization():
     a ^= 1;
     """
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "ASSIGN_ADD" for t in tokens), "Missing '+=' token"
+        assert any(t[0] == "ASSIGN_MUL" for t in tokens), "Missing '*=' token"
+        assert any(t[0] == "ASSIGN_DIV" for t in tokens), "Missing '/=' token"
+        assert any(t[0] == "ASSIGN_SUB" for t in tokens), "Missing '-=' token"
+        assert any(t[0] == "ASSIGN_MOD" for t in tokens), "Missing '%=' token"
+        assert any(t[0] == "ASSIGN_AND" for t in tokens), "Missing '&=' token"
+        assert any(t[0] == "ASSIGN_OR" for t in tokens), "Missing '|=' token"
+        assert any(t[0] == "ASSIGN_XOR" for t in tokens), "Missing '^=' token"
     except SyntaxError:
         pytest.fail("Assignment operators tokenization not implemented.")
 
@@ -212,12 +243,18 @@ def test_const_tokenization():
     code = """
     const int a;
     """
-
     try:
-        tokenize_code(code)
+        tokens = tokenize_code(code)
+        assert any(t[0] == "CONST" for t in tokens), "Missing 'CONST' token"
     except SyntaxError:
         pytest.fail("Const keyword tokenization failed")
 
 
+def test_illegal_character():
+    code = "int a = 1 @#"
+    with pytest.raises(SyntaxError):
+        tokenize_code(code)
+
+
 if __name__ == "__main__":
     pytest.main()