Repository: tianocore/edk2-pytool-library

Issue:
10.7 breaks certain "in" conditionals
This conditional is broken with latest release:
!if ("GCC49" in $(TOOL_CHAIN_TAG)) OR ("GCC5" in $(TOOL_CHAIN_TAG))
...
!endif

Comments:

Patch:
diff --git a/edk2toollib/uefi/edk2/parsers/base_parser.py b/edk2toollib/uefi/edk2/parsers/base_parser.py
index 99b25e2..f22b6c5 100644
--- a/edk2toollib/uefi/edk2/parsers/base_parser.py
+++ b/edk2toollib/uefi/edk2/parsers/base_parser.py
@@ -133,7 +133,16 @@ def ComputeResult(self, value, cond, value2):
         """
         ivalue = value
         ivalue2 = value2
+        if isinstance(value, str):
+            ivalue = value.strip("\"")
+        if isinstance(value2, str):
+            ivalue2 = value2.strip("\"")
+
         # convert it to interpretted value
+        if (cond.upper() == "IN"):
+            # strip quotes
+            self.Logger.debug(f"{ivalue} in {ivalue2}")
+            return ivalue in ivalue2
 
         try:
             ivalue = self.ConvertToInt(ivalue)
@@ -147,6 +156,12 @@ def ComputeResult(self, value, cond, value2):
         except ValueError:
             pass
 
+        # First check our boolean operators
+        if (cond.upper() == "OR"):
+            return ivalue or ivalue2
+        if (cond.upper() == "AND"):
+            return ivalue and ivalue2
+
         # check our truthyness
         if(cond == "=="):
             # equal
@@ -156,17 +171,13 @@ def ComputeResult(self, value, cond, value2):
             # not equal
             return (ivalue != ivalue2) and (value != value2)
 
-        elif (cond.lower() == "in"):
-            # contains
-            return value in value2
-
         # check to make sure we only have digits from here on out
-        if not str.isdigit(value):
+        if not isinstance(value, int) and not str.isdigit(value):
             self.Logger.error(f"{self.__class__}: Unknown value: {value} {ivalue.__class__}")
             self.Logger.debug(f"{self.__class__}: Conditional: {value} {cond}{value2}")
             raise ValueError("Unknown value")
 
-        if not str.isdigit(value2):
+        if not isinstance(value2, int) and not str.isdigit(value2):
             self.Logger.error(f"{self.__class__}: Unknown value: {value2} {ivalue2}")
             self.Logger.debug(f"{self.__class__}: Conditional: {value} {cond} {value2}")
             raise ValueError("Unknown value")
@@ -200,6 +211,8 @@ def ConvertToInt(self, value):
         Returns:
 
         """
+        if isinstance(value, int):
+            return value
         if isinstance(value, str) and value.upper() == "TRUE":
             return 1
         elif isinstance(value, str) and value.upper() == "FALSE":
@@ -320,17 +333,7 @@ def ProcessConditional(self, text):
         else:
             tokens = text.split()
         if(tokens[0].lower() == "!if"):
-            # need to add support for OR/AND
-            if (len(tokens) == 2):
-                value = self.ConvertToInt(tokens[1].strip())
-                self.PushConditional(value == 1)  # if the value is true
-            # we can have tokens in 4, 8, 12 etc
-            elif len(tokens) >= 4 and len(tokens) % 4 == 0:
-                con = self.ComputeResult(tokens[1].strip(), tokens[2].strip(), tokens[3].strip())
-                self.PushConditional(con)
-            else:
-                self.Logger.error("!if conditionals need to be formatted correctly (spaces between each token)")
-                raise RuntimeError("Invalid conditional", text)
+            self.PushConditional(self.EvaluateConditional(text))
             return True
 
         elif(tokens[0].lower() == "!ifdef"):
@@ -365,9 +368,135 @@ def ProcessConditional(self, text):
 
         return False
 
+    def EvaluateConditional(self, text):
+        ''' Uses a pushdown resolver '''
+        text = str(text).strip()
+        if not text.lower().startswith("!if "):
+            raise RuntimeError(f"Invalid conditional cannot be validated: {text}")
+        text = text[3:].strip()
+        self.Logger.debug(f"STAGE 1: {text}")
+        text = self.ReplaceVariables(text)
+        self.Logger.debug(f"STAGE 2: {text}")
+
+        # TOKENIZER
+        # first we create tokens
+        TEXT_MODE = 0
+        QUOTE_MODE = 1
+        MACRO_MODE = 2
+        token = ""
+        mode = 0
+        tokens = []
+        for character in text:
+
+            if character == "\"" and len(token) == 0:
+                mode = QUOTE_MODE
+            elif character == "\"" and mode == QUOTE_MODE:
+                if len(token) > 0:
+                    tokens.append(f"\"{token}\"")
+                    token = ""
+                mode = TEXT_MODE
+            elif character == "$" and len(token) == 0:
+                token += character
+                mode = MACRO_MODE
+            elif character == ')' and mode == MACRO_MODE:
+                token += character
+                tokens.append(token)
+                token = ""
+                mode = TEXT_MODE
+            elif mode == TEXT_MODE and (character == "(" or character == ")"):
+                if len(token) > 0:
+                    tokens.append(token)
+                    token = ""
+                tokens.append(character)
+            elif character == " " and (mode == TEXT_MODE or mode == MACRO_MODE):
+                if len(token) > 0:
+                    tokens.append(token)
+                    token = ""
+                    mode = TEXT_MODE
+            else:
+                token += character
+        # make sure to add in the last token just in case
+        if len(token) > 0:
+            tokens.append(token)
+
+        self.Logger.debug(f"STAGE 3: {' '.join(tokens)}")
+
+        operators = ["OR", "AND", "IN", "==", "!=", ">", "<", "<=", ">="]
+
+        # then we do the lexer and convert operands as necessary
+        for index in range(len(tokens)):
+            token = tokens[index]
+            token_upper = token.upper()
+            if token_upper in operators:
+                token = token_upper
+            elif token_upper == "||":
+                token = "OR"
+            elif token_upper == "&&":
+                token = "AND"
+            elif token_upper == "EQ":
+                token = "=="
+            elif token_upper == "NE":
+                token = "!="
+            tokens[index] = token
+        self.Logger.debug(f"STAGE 4: {tokens}")
+
+        # now we convert in fix into post fix?
+        stack = ["("]
+        tokens.append(")")  # add an extra parathesis
+        expression = []
+        for token in tokens:
+            if token == "(":
+                stack.append(token)
+            elif token == ")":
+                while len(stack) > 0 and stack[-1] != '(':
+                    expression.append(stack.pop())
+            elif token in operators:
+                while len(stack) > 0 and stack[-1] != '(':
+                    self.Logger.debug(stack[-1])
+                    expression.append(stack.pop())
+                stack.append(token)
+            else:
+                expression.append(token)
+        while len(stack) > 0:
+            val = stack.pop()
+            if val != '(':
+                expression.append(val)
+
+        self.Logger.debug(f"STAGE 5: {expression}")
+
+        # Now we evaluate the post fix expression
+        if len(expression) == 0:
+            raise RuntimeError(f"Malformed !if conditional expression {text} {expression}")
+        while len(expression) != 1:
+            first_operand_index = -1
+            for index, item in enumerate(expression):
+                if item in operators:
+                    first_operand_index = index
+                    break
+            if first_operand_index == -1:
+                raise RuntimeError(f"We didn't find an operator to execute in {expression}: {text}")
+            operand = expression[first_operand_index]
+            if first_operand_index < 2:
+                raise RuntimeError(f"We have a stray operand {operand}")
+            operator1 = expression[first_operand_index - 2]
+            operator2 = expression[first_operand_index - 1]
+
+            result = self.ComputeResult(operator1, operand, operator2)
+            self.Logger.debug(f"{operator1} {operand} {operator2} = {result} @ {first_operand_index}")
+            new_expression = expression[:first_operand_index - 2] if first_operand_index > 2 else []
+            self.Logger.debug(new_expression)
+            new_expression += [result, ] + expression[first_operand_index + 1:]
+            expression = new_expression
+
+        final = self.ConvertToInt(expression[0])
+        self.Logger.debug(f" FINAL {expression} {final}")
+
+        return bool(final)
+
     #
     # returns true or false depending on what state of conditional you are currently in
     #
+
     def InActiveCode(self):
         """ """
         ret = True


Test patch:
diff --git a/edk2toollib/uefi/edk2/parsers/base_parser_test.py b/edk2toollib/uefi/edk2/parsers/base_parser_test.py
index a043cec..d51807a 100644
--- a/edk2toollib/uefi/edk2/parsers/base_parser_test.py
+++ b/edk2toollib/uefi/edk2/parsers/base_parser_test.py
@@ -49,6 +49,9 @@ def test_replace_macro_local_var_priority(self):
         line = "Hello $(name)!"
         self.assertEqual(parser.ReplaceVariables(line), "Hello fred!")
 
+
+class TestBaseParserConditionals(unittest.TestCase):
+
     def test_replace_macro_without_resolution(self):
         parser = BaseParser("")
         parser.SetInputVars({
@@ -385,6 +388,52 @@ def test_process_conditional_reset(self):
         self.assertTrue(parser.InActiveCode())
         self.assertEqual(len(parser.ConditionalStack), 0)
 
+    def test_process_in_conditional(self):
+        parser = BaseParser("")
+        parser.SetInputVars({"TOOL_CHAIN_TAG": "GCC5_TEST"})
+        self.assertTrue(parser.ProcessConditional(
+            '!if ("GCC49" in $(TOOL_CHAIN_TAG)) OR ("GCC5" in $(TOOL_CHAIN_TAG))'))
+        self.assertTrue(parser.InActiveCode())
+        parser.ResetParserState()
+        parser.SetInputVars({"TOOL_CHAIN_TAG": "TESTGCC49"})
+        self.assertTrue(parser.ProcessConditional(
+            '!if ("GCC49" in $(TOOL_CHAIN_TAG)) OR ("GCC5" in $(TOOL_CHAIN_TAG))'))
+        self.assertTrue(parser.InActiveCode())
+        parser.ResetParserState()
+        # Don't give it a tool chain tag that isn't in the things we're searching for
+        parser.SetInputVars({"TOOL_CHAIN_TAG": "NOTFOUND"})
+        self.assertTrue(parser.ProcessConditional(
+            '!if ("GCC49" in $(TOOL_CHAIN_TAG)) OR ("GCC5" in $(TOOL_CHAIN_TAG))'))
+        self.assertFalse(parser.InActiveCode())
+
+    def test_process_or_operation_conditional(self):
+        parser = BaseParser("")
+        self.assertTrue(parser.EvaluateConditional('!IF TRUE OR FALSE'))
+        self.assertTrue(parser.EvaluateConditional('!if FALSE OR TRUE'))
+        self.assertTrue(parser.EvaluateConditional('!if FALSE || TRUE'))
+        self.assertTrue(parser.EvaluateConditional('!if TRUE OR TRUE'))
+        self.assertFalse(parser.EvaluateConditional('!if FALSE OR FALSE'))
+        self.assertFalse(parser.EvaluateConditional('!if FALSE || FALSE'))
+
+    def test_process_and_operation_conditional(self):
+        parser = BaseParser("")
+        self.assertFalse(parser.EvaluateConditional('!if TRUE AND FALSE'))
+        self.assertFalse(parser.EvaluateConditional('!if FALSE AND TRUE'))
+        self.assertTrue(parser.EvaluateConditional('!if TRUE AND TRUE'))
+        self.assertTrue(parser.EvaluateConditional('!if TRUE && TRUE'))
+        self.assertFalse(parser.EvaluateConditional('!if FALSE AND FALSE'))
+        self.assertFalse(parser.EvaluateConditional('!if FALSE && FALSE'))
+
+    def test_process_invalid_conditional(self):
+        parser = BaseParser("")
+        with self.assertRaises(RuntimeError):
+            parser.EvaluateConditional('!if TRUE AND FALSE AND')
+        with self.assertRaises(RuntimeError):
+            parser.EvaluateConditional('TRUE AND FALSE AND')
+
+
+class TestBaseParserGuids(unittest.TestCase):
+
     def test_is_guid(self):
         guid1 = "= { 0xD3B36F2C, 0xD551, 0x11D4, {0x9A, 0x46, 0x0, 0x90, 0x27, 0x3F, 0xC1,0xD }}"
         parser = BaseParser("")
@@ -421,6 +470,9 @@ def test_parse_guid(self):
         guid4_result = parser.ParseGuid(guid4)
         self.assertEqual(guid4_result, guid4_answer)
 
+
+class TestBaseParserVariables(unittest.TestCase):
+
     def test_replace_input_variables(self):
         parser = BaseParser("")
         variables = {
@@ -477,6 +529,9 @@ def test_replace_local_variables(self):
             val = "var " + str(variables[variable_key])
             self.assertEqual(result, val)
 
+
+class TestBaseParserPathAndFile(unittest.TestCase):
+
     # because of how this works we use WriteLines, SetAbsPath, and SetPackagePath
     def test_find_path(self):
         # we're using write lines to make sure everything wo