Repository: CrossGL/crosstl

Issue:
Incorrect Regular Expression Patterns for WHILE and DO Tokens
Description: The regular expression patterns for the WHILE and DO tokens in the lexer are incorrect due to unnecessary escape characters (\) before the keywords. This could cause the lexer to fail in matching these keywords properly, leading to errors in tokenization.

Current Patterns:
`("WHILE", r"\b\while\b"),`
`("DO", r"\b\do\b"),`

Suggested Fix: Remove the backslashes from the WHILE and DO regular expressions to correct the token matching in [crosstl/backend/DirectX/DirectxLexer.py](url)

@NripeshN would you please assign this to me ?

Discussion:
@tiashamaitra: @CrossGL-issue-bot assign me@CrossGL-issue-bot: The issue you are trying to assign to yourself is already assigned.@AeleSahithi: Hi @NripeshN,

I’ve submitted PR (https://github.com/CrossGL/crosstl/pull/251), and all checks have passed successfully.
 please review the pull request 

Thank you!

Solution patch:
diff --git a/crosstl/backend/DirectX/DirectxLexer.py b/crosstl/backend/DirectX/DirectxLexer.py
index b6003c1d..88eadcce 100644
--- a/crosstl/backend/DirectX/DirectxLexer.py
+++ b/crosstl/backend/DirectX/DirectxLexer.py
@@ -21,8 +21,8 @@
     ("ELSE_IF", r"\belse\sif\b"),
     ("ELSE", r"\belse\b"),
     ("FOR", r"\bfor\b"),
-    ("WHILE", r"\b\while\b"),
-    ("DO", r"\b\do\b"),
+    ("WHILE", r"\bwhile\b"),
+    ("DO", r"\bdo\b"),
     ("REGISTER", r"\bregister\b"),
     ("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_]*"),
     ("NUMBER", r"\d+(\.\d+)?"),
@@ -67,6 +67,7 @@
     ("CASE", r"\bcase\b"),
     ("DEFAULT", r"\bdefault\b"),
     ("BREAK", r"\bbreak\b"),
+    ("MOD", r"%"),
 ]
 
 KEYWORDS = {
diff --git a/crosstl/backend/DirectX/DirectxParser.py b/crosstl/backend/DirectX/DirectxParser.py
index e7c03095..5c5ff80b 100644
--- a/crosstl/backend/DirectX/DirectxParser.py
+++ b/crosstl/backend/DirectX/DirectxParser.py
@@ -498,7 +498,7 @@ def parse_additive(self):
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/backend/Metal/MetalLexer.py b/crosstl/backend/Metal/MetalLexer.py
index d6701235..8c0967dd 100644
--- a/crosstl/backend/Metal/MetalLexer.py
+++ b/crosstl/backend/Metal/MetalLexer.py
@@ -64,6 +64,8 @@
     ("EQUALS", r"="),
     ("bitwise_and", r"&"),
     ("WHITESPACE", r"\s+"),
+    ("MOD", r"%"),
+    ("ASSIGN_MOD", r"%="),
 ]
 
 KEYWORDS = {
diff --git a/crosstl/backend/Metal/MetalParser.py b/crosstl/backend/Metal/MetalParser.py
index f72a7bf0..d121507b 100644
--- a/crosstl/backend/Metal/MetalParser.py
+++ b/crosstl/backend/Metal/MetalParser.py
@@ -452,7 +452,7 @@ def parse_additive(self):
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/backend/Mojo/MojoLexer.py b/crosstl/backend/Mojo/MojoLexer.py
index 355b8e9e..b1395ec3 100644
--- a/crosstl/backend/Mojo/MojoLexer.py
+++ b/crosstl/backend/Mojo/MojoLexer.py
@@ -50,6 +50,7 @@
     ("DOT", r"\."),
     ("EQUALS", r"="),
     ("WHITESPACE", r"\s+"),
+    ("MOD", r"%"),
 ]
 
 # Define keywords specific to mojo
diff --git a/crosstl/backend/Mojo/MojoParser.py b/crosstl/backend/Mojo/MojoParser.py
index 64cf50eb..e2ac3031 100644
--- a/crosstl/backend/Mojo/MojoParser.py
+++ b/crosstl/backend/Mojo/MojoParser.py
@@ -460,7 +460,7 @@ def parse_additive(self):
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/backend/Opengl/OpenglParser.py b/crosstl/backend/Opengl/OpenglParser.py
index f2d15ecc..370334cb 100644
--- a/crosstl/backend/Opengl/OpenglParser.py
+++ b/crosstl/backend/Opengl/OpenglParser.py
@@ -712,7 +712,7 @@ def parse_multiplicative(self):
 
         """
         expr = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[0]
             self.eat(op)
             right = self.parse_unary()
diff --git a/crosstl/backend/Vulkan/VulkanLexer.py b/crosstl/backend/Vulkan/VulkanLexer.py
index 471ab723..4b2e5d8b 100644
--- a/crosstl/backend/Vulkan/VulkanLexer.py
+++ b/crosstl/backend/Vulkan/VulkanLexer.py
@@ -36,7 +36,6 @@
     ("MINUS", r"-"),
     ("MULTIPLY", r"\*"),
     ("DIVIDE", r"/"),
-    ("MODULUS", r"%"),
     ("LESS_EQUAL", r"<="),
     ("GREATER_EQUAL", r">="),
     ("NOT_EQUAL", r"!="),
@@ -50,6 +49,7 @@
     ("BINARY_NOT", r"~"),
     ("QUESTION", r"\?"),
     ("COLON", r":"),
+    ("MOD", r"%"),
 ]
 
 KEYWORDS = {
diff --git a/crosstl/backend/Vulkan/VulkanParser.py b/crosstl/backend/Vulkan/VulkanParser.py
index 8b2247cd..f72d68d0 100644
--- a/crosstl/backend/Vulkan/VulkanParser.py
+++ b/crosstl/backend/Vulkan/VulkanParser.py
@@ -466,9 +466,9 @@ def parse_primary(self):
 
     def parse_multiplicative(self):
         left = self.parse_primary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
-            op = self.current_token[0]
-            self.eat(op)
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
+            op = self.current_token[1]
+            self.eat(self.current_token[0])
             right = self.parse_primary()
             left = BinaryOpNode(left, op, right)
         return left
diff --git a/crosstl/backend/slang/SlangLexer.py b/crosstl/backend/slang/SlangLexer.py
index 13d5a7e6..9d8808b1 100644
--- a/crosstl/backend/slang/SlangLexer.py
+++ b/crosstl/backend/slang/SlangLexer.py
@@ -72,6 +72,8 @@
     ("CONSTEXPR", r"\bconstexpr\b"),
     ("STATIC", r"\bstatic\b"),
     ("INLINE", r"\binline\b"),
+    ("MOD", r"%"),  # Add modulus operator
+    # Add modulus assignment
 ]
 
 # Keywords map for matching identifiers to token types
diff --git a/crosstl/backend/slang/SlangParser.py b/crosstl/backend/slang/SlangParser.py
index 0e097793..daa5200d 100644
--- a/crosstl/backend/slang/SlangParser.py
+++ b/crosstl/backend/slang/SlangParser.py
@@ -489,7 +489,7 @@ def parse_additive(self):
 
     def parse_multiplicative(self):
         left = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[1]
             self.eat(self.current_token[0])
             right = self.parse_unary()
diff --git a/crosstl/translator/lexer.py b/crosstl/translator/lexer.py
index ac89f534..21929417 100644
--- a/crosstl/translator/lexer.py
+++ b/crosstl/translator/lexer.py
@@ -1,6 +1,7 @@
 import re
 from collections import OrderedDict
 
+
 TOKENS = OrderedDict(
     [
         ("COMMENT_SINGLE", r"//.*"),
@@ -78,6 +79,7 @@
     ]
 )
 
+
 KEYWORDS = {
     "shader": "SHADER",
     "void": "VOID",
diff --git a/crosstl/translator/parser.py b/crosstl/translator/parser.py
index d9cbccfa..c17f72a5 100644
--- a/crosstl/translator/parser.py
+++ b/crosstl/translator/parser.py
@@ -653,7 +653,7 @@ def parse_multiplicative(self):
 
         """
         expr = self.parse_unary()
-        while self.current_token[0] in ["MULTIPLY", "DIVIDE"]:
+        while self.current_token[0] in ["MULTIPLY", "DIVIDE", "MOD"]:
             op = self.current_token[0]
             self.eat(op)
             right = self.parse_unary()
@@ -803,6 +803,7 @@ def parse_expression(self):
             "BITWISE_AND",
             "ASSIGN_SHIFT_RIGHT",
             "ASSIGN_SHIFT_LEFT",
+            "MOD",
         ]:
             op = self.current_token[0]
             self.eat(op)


Test patch:
diff --git a/tests/test_backend/test_directx/test_lexer.py b/tests/test_backend/test_directx/test_lexer.py
index aa785b9e..83b3825a 100644
--- a/tests/test_backend/test_directx/test_lexer.py
+++ b/tests/test_backend/test_directx/test_lexer.py
@@ -250,5 +250,21 @@ def test_double_dtype_tokenization():
         pytest.fail("double dtype tokenization is not implemented.")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_directx/test_parser.py b/tests/test_backend/test_directx/test_parser.py
index efb59d2d..9460ca04 100644
--- a/tests/test_backend/test_directx/test_parser.py
+++ b/tests/test_backend/test_directx/test_parser.py
@@ -355,5 +355,18 @@ def test_double_dtype_parsing():
         pytest.fail("double dtype not implemented.")
 
 
+def test_mod_parsing():
+    code = """
+    void main() {
+        int a = 10 % 3;  // Basic modulus
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_metal/test_lexer.py b/tests/test_backend/test_metal/test_lexer.py
index eaaf4b57..e745ca77 100644
--- a/tests/test_backend/test_metal/test_lexer.py
+++ b/tests/test_backend/test_metal/test_lexer.py
@@ -126,5 +126,21 @@ def test_if_else_tokenization():
         pytest.fail("If-else statement tokenization not implemented.")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_metal/test_parser.py b/tests/test_backend/test_metal/test_parser.py
index c30c416f..02286b72 100644
--- a/tests/test_backend/test_metal/test_parser.py
+++ b/tests/test_backend/test_metal/test_parser.py
@@ -153,5 +153,19 @@ def test_if_else():
         pytest.fail("If-else statement parsing not implemented.")
 
 
+def test_mod_parsing():
+    code = """
+    fragment float4 fragmentMain() {
+        int a = 10 % 3;  // Basic modulus
+        return float4(1.0);
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_mojo/test_lexer.py b/tests/test_backend/test_mojo/test_lexer.py
index 6c249050..e23afcef 100644
--- a/tests/test_backend/test_mojo/test_lexer.py
+++ b/tests/test_backend/test_mojo/test_lexer.py
@@ -9,9 +9,20 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
 
 
 if __name__ == "__main__":
diff --git a/tests/test_backend/test_mojo/test_parser.py b/tests/test_backend/test_mojo/test_parser.py
index 36953cdc..295781b1 100644
--- a/tests/test_backend/test_mojo/test_parser.py
+++ b/tests/test_backend/test_mojo/test_parser.py
@@ -22,9 +22,16 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_parsing():
+    code = """
+    fn main():
+        let a: Int = 10 % 3  # Basic modulus
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_backend/test_opengl/test_lexer.py b/tests/test_backend/test_opengl/test_lexer.py
index fa72f512..a1129290 100644
--- a/tests/test_backend/test_opengl/test_lexer.py
+++ b/tests/test_backend/test_opengl/test_lexer.py
@@ -123,5 +123,21 @@ def test_double_dtype_tokenization():
         pytest.fail("double tokenization not implemented")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_opengl/test_parser.py b/tests/test_backend/test_opengl/test_parser.py
index 661ccc25..79a107b5 100644
--- a/tests/test_backend/test_opengl/test_parser.py
+++ b/tests/test_backend/test_opengl/test_parser.py
@@ -197,5 +197,18 @@ def test_double_dtype_tokenization():
         pytest.fail("double tokenization not implemented")
 
 
+def test_mod_parsing():
+    code = """
+    void main() {
+        int a = 10 % 3;  // Basic modulus
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_slang/test_lexer.py b/tests/test_backend/test_slang/test_lexer.py
index adf8b744..a9bbd989 100644
--- a/tests/test_backend/test_slang/test_lexer.py
+++ b/tests/test_backend/test_slang/test_lexer.py
@@ -101,5 +101,21 @@ def test_function_call_tokenization():
         pytest.fail("Function call tokenization not implemented.")
 
 
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_slang/test_parser.py b/tests/test_backend/test_slang/test_parser.py
index 34976622..69fd232d 100644
--- a/tests/test_backend/test_slang/test_parser.py
+++ b/tests/test_backend/test_slang/test_parser.py
@@ -117,5 +117,21 @@ def test_function_call_parsing():
         pytest.fail("function call parsing not implemented.")
 
 
+def test_mod_parsing():
+    code = """
+    [shader("vertex")]
+    VertexStageOutput vertexMain(AssembledVertex assembledVertex) {
+        VertexStageOutput output;
+        int a = 10 % 3;  // Basic modulus
+        return output;
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/tests/test_backend/test_vulkan/test_lexer.py b/tests/test_backend/test_vulkan/test_lexer.py
index 82221481..9fabf09a 100644
--- a/tests/test_backend/test_vulkan/test_lexer.py
+++ b/tests/test_backend/test_vulkan/test_lexer.py
@@ -9,9 +9,20 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_tokenization():
+    code = """
+        int a = 10 % 3;  // Basic modulus
+    """
+    tokens = tokenize_code(code)
+
+    # Find the modulus operator in tokens
+    has_mod = False
+    for token in tokens:
+        if token == ("MOD", "%"):
+            has_mod = True
+            break
+
+    assert has_mod, "Modulus operator (%) not tokenized correctly"
 
 
 if __name__ == "__main__":
diff --git a/tests/test_backend/test_vulkan/test_parser.py b/tests/test_backend/test_vulkan/test_parser.py
index 736e78ba..d1af630c 100644
--- a/tests/test_backend/test_vulkan/test_parser.py
+++ b/tests/test_backend/test_vulkan/test_parser.py
@@ -22,9 +22,18 @@ def tokenize_code(code: str) -> List:
     return lexer.tokens
 
 
-# ToDO: Implement the tests
-def test_struct():
-    pass
+def test_mod_parsing():
+    code = """
+    
+    void main() {
+        int a = 10 % 3;  // Basic modulus
+    }
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operator parsing not implemented")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_translator/test_parser.py b/tests/test_translator/test_parser.py
index 8f498322..8f3ca4a9 100644
--- a/tests/test_translator/test_parser.py
+++ b/tests/test_translator/test_parser.py
@@ -586,5 +586,40 @@ def test_and_operator():
         pytest.fail("Bitwise AND not working")
 
 
+def test_modulus_operations():
+    code = """
+    shader main {
+    struct VSInput {
+        vec2 texCoord @ TEXCOORD0;
+    };
+    struct VSOutput {
+        vec4 color @ COLOR;
+    };
+    sampler2D iChannel0;
+    vertex {
+        VSOutput main(VSInput input) {
+            VSOutput output;
+            // Test modulus operations
+            int value = 1200;
+            value = value % 10;      // Basic modulus
+            value %= 5;              // Modulus assignment
+            output.color = vec4(float(value) / 10.0, 0.0, 0.0, 1.0);
+            return output;
+        }
+    }
+    fragment {
+        vec4 main(VSOutput input) @ gl_FragColor {
+            return vec4(input.color.rgb, 1.0);
+        }
+    }
+}
+    """
+    try:
+        tokens = tokenize_code(code)
+        parse_code(tokens)
+    except SyntaxError:
+        pytest.fail("Modulus operations not working")
+
+
 if __name__ == "__main__":
     pytest.main()