Repository: CrossGL/crosstl

Issue:
 Incorrect Regular Expression Patterns for WHILE and DO Tokens
Description: The regular expression patterns for the WHILE and DO tokens in the lexer are incorrect due to unnecessary escape characters (\) before the keywords. This could cause the lexer to fail in matching these keywords properly, leading to errors in tokenization.

Current Patterns:
`("WHILE", r"\b\while\b"),`
`("DO", r"\b\do\b"),`

Suggested Fix: Remove the backslashes from the WHILE and DO regular expressions to correct the token matching in [crosstl/backend/DirectX/DirectxLexer.py](url)

@NripeshN would you please assign this to me ?


Discussion:
AeleSahithi: 
@NripeshN, please review this pull request 

Patch:
diff --git a/crosstl/backend/DirectX/DirectxLexer.py b/crosstl/backend/DirectX/DirectxLexer.py
index b6003c1..88eadcc 100644
--- a/crosstl/backend/DirectX/DirectxLexer.py
+++ b/crosstl/backend/DirectX/DirectxLexer.py
@@ -21,8 +21,8 @@ TOKENS = [
     ("ELSE_IF", r"\belse\sif\b"),
     ("ELSE", r"\belse\b"),
     ("FOR", r"\bfor\b"),
-    ("WHILE", r"\b\while\b"),
-    ("DO", r"\b\do\b"),
+    ("WHILE", r"\bwhile\b"),
+    ("DO", r"\bdo\b"),
     ("REGISTER", r"\bregister\b"),
     ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
     ("NUMBER", r"\d+(\.\d+)?"),
@@ -67,6 +67,7 @@ TOKENS = [
     ("CASE", r"\bcase\b"),
     ("DEFAULT", r"\bdefault\b"),
     ("BREAK", r"\bbreak\b"),
+    ("MOD", r"%"),
 ]
 
 KEYWORDS = {
diff --git a/crosstl/backend/DirectX/DirectxParser.py b/crosstl/backend/DirectX/DirectxParser.py
index e7c0309..5c5ff80 100644
--- a/crosstl/backend/DirectX/DirectxParser.py
+++ b/crosstl/backend/DirectX/DirectxParser.py
@@ -498,7 +498,7 @@ class HLSLParser:
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/backend/Metal/MetalLexer.py b/crosstl/backend/Metal/MetalLexer.py
index d670123..8c0967d 100644
--- a/crosstl/backend/Metal/MetalLexer.py
+++ b/crosstl/backend/Metal/MetalLexer.py
@@ -64,6 +64,8 @@ TOKENS = [
     ("EQUALS", r"="),
     ("bitwise_and", r"&"),
     ("WHITESPACE", r"\s+"),
+    ("MOD", r"%"),
+    ("ASSIGN_MOD", r"%="),
 ]
 
 KEYWORDS = {
diff --git a/crosstl/backend/Metal/MetalParser.py b/crosstl/backend/Metal/MetalParser.py
index f72a7bf..d121507 100644
--- a/crosstl/backend/Metal/MetalParser.py
+++ b/crosstl/backend/Metal/MetalParser.py
@@ -452,7 +452,7 @@ class MetalParser:
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/backend/Mojo/MojoLexer.py b/crosstl/backend/Mojo/MojoLexer.py
index 355b8e9..b1395ec 100644
--- a/crosstl/backend/Mojo/MojoLexer.py
+++ b/crosstl/backend/Mojo/MojoLexer.py
@@ -50,6 +50,7 @@ TOKENS = [
     ("DOT", r"\."),
     ("EQUALS", r"="),
     ("WHITESPACE", r"\s+"),
+    ("MOD", r"%"),
 ]
 
 # Define keywords specific to mojo
diff --git a/crosstl/backend/Mojo/MojoParser.py b/crosstl/backend/Mojo/MojoParser.py
index 64cf50e..e2ac303 100644
--- a/crosstl/backend/Mojo/MojoParser.py
+++ b/crosstl/backend/Mojo/MojoParser.py
@@ -460,7 +460,7 @@ class MojoParser:
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/backend/Opengl/OpenglParser.py b/crosstl/backend/Opengl/OpenglParser.py
index f2d15ec..370334c 100644
--- a/crosstl/backend/Opengl/OpenglParser.py
+++ b/crosstl/backend/Opengl/OpenglParser.py
@@ -712,7 +712,7 @@ class GLSLParser:
 
         """
         expr = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[0]
             self.eat(op)
             right = self.parse_unary()
diff --git a/crosstl/backend/Vulkan/VulkanLexer.py b/crosstl/backend/Vulkan/VulkanLexer.py
index 471ab72..4b2e5d8 100644
--- a/crosstl/backend/Vulkan/VulkanLexer.py
+++ b/crosstl/backend/Vulkan/VulkanLexer.py
@@ -36,7 +36,6 @@ TOKENS = [
     ("MINUS", r"-"),
     ("MULTIPLY", r"\*"),
     ("DIVIDE", r"/"),
-    ("MODULUS", r"%"),
     ("LESS_EQUAL", r"<="),
     ("GREATER_EQUAL", r">="),
     ("NOT_EQUAL", r"!="),
@@ -50,6 +49,7 @@ TOKENS = [
     ("BINARY_NOT", r"~"),
     ("QUESTION", r"\?"),
     ("COLON", r":"),
+    ("MOD", r"%"),
 ]
 
 KEYWORDS = {
diff --git a/crosstl/backend/Vulkan/VulkanParser.py b/crosstl/backend/Vulkan/VulkanParser.py
index 8b2247c..f72d68d 100644
--- a/crosstl/backend/Vulkan/VulkanParser.py
+++ b/crosstl/backend/Vulkan/VulkanParser.py
@@ -466,9 +466,9 @@ class VulkanParser:
 
     def parse_multiplicative(self):
         left = self.parse_primary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
-            op = self.current_token[0]
-            self.eat(op)
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
+            op = self.current_token[1]
+            self.eat(self.current_token[0])
             right = self.parse_primary()
             left = BinaryOpNode(left, op, right)
         return left
diff --git a/crosstl/backend/slang/SlangLexer.py b/crosstl/backend/slang/SlangLexer.py
index 13d5a7e..9d8808b 100644
--- a/crosstl/backend/slang/SlangLexer.py
+++ b/crosstl/backend/slang/SlangLexer.py
@@ -72,6 +72,8 @@ TOKENS = [
     ("CONSTEXPR", r"\bconstexpr\b"),
     ("STATIC", r"\bstatic\b"),
     ("INLINE", r"\binline\b"),
+    ("MOD", r"%"),  # Add modulus operator
+    # Add modulus assignment
 ]
 
 # Keywords map for matching identifiers to token types
diff --git a/crosstl/backend/slang/SlangParser.py b/crosstl/backend/slang/SlangParser.py
index 0e09779..daa5200 100644
--- a/crosstl/backend/slang/SlangParser.py
+++ b/crosstl/backend/slang/SlangParser.py
@@ -489,7 +489,7 @@ class SlangParser:
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/translator/lexer.py b/crosstl/translator/lexer.py
index ac89f53..2192941 100644
--- a/crosstl/translator/lexer.py
+++ b/crosstl/translator/lexer.py
@@ -1,6 +1,7 @@
 import re
 from collections import OrderedDict
 
+
 TOKENS = OrderedDict(
     [
         ("COMMENT_SINGLE", r"//.*"),
@@ -78,6 +79,7 @@ TOKENS = OrderedDict(
     ]
 )
 
+
 KEYWORDS = {
     "shader": "SHADER",
     "void": "VOID",
diff --git a/crosstl/translator/parser.py b/crosstl/translator/parser.py
index d9cbccf..c17f72a 100644
--- a/crosstl/translator/parser.py
+++ b/crosstl/translator/parser.py
@@ -653,7 +653,7 @@ class Parser:
 
         """
         expr = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[0]
             self.eat(op)
             right = self.parse_unary()
@@ -803,6 +803,7 @@ class Parser:
             "BITWISE_AND",
             "ASSIGN_SHIFT_RIGHT",
             "ASSIGN_SHIFT_LEFT",
+            "MOD",
         ]:
             op = self.current_token[0]
             self.eat(op)


Test patch:
diff --git a/tests/test_backend/test_directx/test_lexer.py b/tests/test_backend/test_directx/test_lexer.py
index aa785b9..83b3825 100644
--- a/tests/test_backend/test_directx/test_lexer.py
+++ b/tests/test_backend/test_directx/test_lexer.py
@@ -250,5 +250,21 @@ def test_double_dtype_tokenization():
         pytest.fail("double dtype tokenization is not implemented.")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_directx/test_parser.py b/tests/test_backend/test_directx/test_parser.py
index efb59d2..9460ca0 100644
--- a/tests/test_backend/test_directx/test_parser.py
+++ b/tests/test_backend/test_directx/test_parser.py
@@ -355,5 +355,18 @@ def test_double_dtype_parsing():
         pytest.fail("double dtype not implemented.")
 
 
+def test_mod_parsing():
+    code = """
+    void main() {
+        int a = 10 % 3;  // Basic modulus
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_metal/test_lexer.py b/tests/test_backend/test_metal/test_lexer.py
index eaaf4b5..e745ca7 100644
--- a/tests/test_backend/test_metal/test_lexer.py
+++ b/tests/test_backend/test_metal/test_lexer.py
@@ -126,5 +126,21 @@ def test_if_else_tokenization():
         pytest.fail("If-else statement tokenization not implemented.")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_metal/test_parser.py b/tests/test_backend/test_metal/test_parser.py
index c30c416..02286b7 100644
--- a/tests/test_backend/test_metal/test_parser.py
+++ b/tests/test_backend/test_metal/test_parser.py
@@ -153,5 +153,19 @@ def test_if_else():
         pytest.fail("If-else statement parsing not implemented.")
 
 
+def test_mod_parsing():
+    code = """
+    fragment float4 fragmentMain() {
+        int a = 10 % 3;  // Basic modulus
+        return float4(1.0);
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_mojo/test_lexer.py b/tests/test_backend/test_mojo/test_lexer.py
index 6c24905..e23afce 100644
--- a/tests/test_backend/test_mojo/test_lexer.py
+++ b/tests/test_backend/test_mojo/test_lexer.py
@@ -9,9 +9,20 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
 
 
 if __name__ == "__main__":
diff --git a/tests/test_backend/test_mojo/test_parser.py b/tests/test_backend/test_mojo/test_parser.py
index 36953cd..295781b 100644
--- a/tests/test_backend/test_mojo/test_parser.py
+++ b/tests/test_backend/test_mojo/test_parser.py
@@ -22,9 +22,16 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_parsing():
+    code = """
+    fn main():
+        let a: Int = 10 % 3  # Basic modulus
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_backend/test_opengl/test_lexer.py b/tests/test_backend/test_opengl/test_lexer.py
index fa72f51..a112929 100644
--- a/tests/test_backend/test_opengl/test_lexer.py
+++ b/tests/test_backend/test_opengl/test_lexer.py
@@ -123,5 +123,21 @@ def test_double_dtype_tokenization():
         pytest.fail("double tokenization not implemented")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_opengl/test_parser.py b/tests/test_backend/test_opengl/test_parser.py
index 661ccc2..79a107b 100644
--- a/tests/test_backend/test_opengl/test_parser.py
+++ b/tests/test_backend/test_opengl/test_parser.py
@@ -197,5 +197,18 @@ def test_double_dtype_tokenization():
         pytest.fail("double tokenization not implemented")
 
 
+def test_mod_parsing():
+    code = """
+    void main() {
+        int a = 10 % 3;  // Basic modulus
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_slang/test_lexer.py b/tests/test_backend/test_slang/test_lexer.py
index adf8b74..a9bbd98 100644
--- a/tests/test_backend/test_slang/test_lexer.py
+++ b/tests/test_backend/test_slang/test_lexer.py
@@ -101,5 +101,21 @@ def test_function_call_tokenization():
         pytest.fail("Function call tokenization not implemented.")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_slang/test_parser.py b/tests/test_backend/test_slang/test_parser.py
index 3497662..69fd232 100644
--- a/tests/test_backend/test_slang/test_parser.py
+++ b/tests/test_backend/test_slang/test_parser.py
@@ -117,5 +117,21 @@ def test_function_call_parsing():
         pytest.fail("function call parsing not implemented.")
 
 
+def test_mod_parsing():
+    code = """
+    [shader("vertex")]
+    VertexStageOutput vertexMain(AssembledVertex assembledVertex) {
+        VertexStageOutput output;
+        int a = 10 % 3;  // Basic modulus
+        return output;
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_vulkan/test_lexer.py b/tests/test_backend/test_vulkan/test_lexer.py
index 8222148..9fabf09 100644
--- a/tests/test_backend/test_vulkan/test_lexer.py
+++ b/tests/test_backend/test_vulkan/test_lexer.py
@@ -9,9 +9,20 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
 
 
 if __name__ == "__main__":
diff --git a/tests/test_backend/test_vulkan/test_parser.py b/tests/test_backend/test_vulkan/test_parser.py
index 736e78b..d1af630 100644
--- a/tests/test_backend/test_vulkan/test_parser.py
+++ b/tests/test_backend/test_vulkan/test_parser.py
@@ -22,9 +22,18 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_parsing():
+    code = """
+    
+    void main() {
+        int a = 10 % 3;  // Basic modulus
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_translator/test_parser.py b/tests/test_translator/test_parser.py
index 8f49832..8f3ca4a 100644
--- a/tests/test_translator/test_parser.py
+++ b/tests/test_translator/test_parser.py
@@ -586,5 +586,40 @@ def test_and_operator():
         pytest.fail("Bitwise AND not working")
 
 
+def test_modulus_operations():
+    code = """
+    shader main {
+    struct VSInput {
+        vec2 texCoord @ TEXCOORD0;
+    };
+    struct VSOutput {
+        vec4 color @ COLOR;
+    };
+    sampler2D iChannel0;
+    vertex {
+        VSOutput main(VSInput input) {
+            VSOutput output;
+            // Test modulus operations
+            int value = 1200;
+            value = value % 10;      // Basic modulus
+            value %= 5;              // Modulus assignment
+            output.color = vec4(float(value) / 10.0, 0.0, 0.0, 1.0);
+            return output;
+        }
+    }
+    fragment {
+        vec4 main(VSOutput input) @ gl_FragColor {
+            return vec4(input.color.rgb, 1.0);
+        }
+    }
+}
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operations not working")
+
+
 if __name__ == "__main__":
     pytest.main()