Repository: d0c-s4vage/pfp (python)

Issue:
Integer promotion problems, part deux
After creating the issue, checkboxes will appear where `[] label` exist in the
markdown. You can check/uncheck them to fill out the environment section.

## Describe the bug

Pfp doesn't correctly handle integer promotion. E.g. this should print `10,896`

```c
local ubyte cur = 7;
local uint test1 = (uint)(10);
local uint test2 = (uint)((cur & 0x7f) << 7);
Printf("%u,%u", test1, test2);
```

Currently, the output in pfp is `10,128`.

## Stack Trace

None

## Expected Behavior

Should have the same integer promotion as C and 010 Editor

## Implementation/Fix Notes/Thoughts

In the statement `local uint test2 = (uint)((cur & 0x7f) << 7);`, `cur` and `0x7f` are treated as `ubyte` types and aren't promoted until the cast occurs. The promotion needs to happen before operations occur, so that the `&` and the `<<` binary operators operate on the target cast type `uint`, and not the original types (`ubyte`)

Pull request:
Target types for integer promotion are now propagated correctly
fixes #120 

Patch:
diff --git a/pfp/fields.py b/pfp/fields.py
index 58810ca..78dd419 100644
--- a/pfp/fields.py
+++ b/pfp/fields.py
@@ -262,6 +262,12 @@ class Field(object):
         if stream is not None:
             self._pfp__parse(stream, save_offset=True)
 
+    def _pfp__get_class(self):
+        """Return the class for this field. This would be used for things like
+        integer promotion and type casting.
+        """
+        return self.__class__
+
     # see #51 - fields should have a method of returning the full path of its name
     def _pfp__path(self):
         """Return the full pathname of this field. E.g. given
@@ -678,6 +684,12 @@ class ImplicitArrayWrapper(Field):
         super(Field, self).__setattr__("last_field", last_field)
         super(Field, self).__setattr__("implicit_array", implicit_array)
 
+    def _pfp__get_class(self):
+        """Return the last field class type and not the array type. Overrides
+        :any:`Field._pfp__get_class`.
+        """
+        return self.last_field._pfp__get_class()
+
     def __setattr__(self, name, value):
         """Custom setattr that forwards all sets to the last_field
         """
diff --git a/pfp/interp.py b/pfp/interp.py
index 75e5369..73aafa6 100644
--- a/pfp/interp.py
+++ b/pfp/interp.py
@@ -398,7 +398,7 @@ class Scope(object):
 
         """
         if new_scope is None:
-            new_scope = {"types": {}, "vars": {}}
+            new_scope = {"types": {}, "vars": {}, "meta": {}}
         self._curr_scope = new_scope
         self._dlog("pushing new scope, scope level = {}".format(self.level()))
         self._scope_stack.append(self._curr_scope)
@@ -428,6 +428,34 @@ class Scope(object):
         self._dlog("popping scope, scope level = {}".format(self.level()))
         self._curr_scope = self._scope_stack[-1]
         return res
+    
+    def clear_meta(self):
+        """Clear metadata about the current statement
+        """
+        self._curr_scope["meta"] = {}
+    
+    def push_meta(self, meta_name, meta_value):
+        """Push metadata about the current statement onto the metadata stack
+        for the current statement. Mostly used for tracking integer promotion
+        and casting types
+        """
+        self._dlog("adding metadata '{}'".format(meta_name))
+        self._curr_scope["meta"].setdefault(meta_name, []).append(meta_value)
+
+    def get_meta(self, meta_name):
+        """Get the current meta value named ``meta_name``
+        """
+        self._dlog("getting metadata '{}'".format(meta_name))
+        return self._curr_scope["meta"].get(meta_name, [None])[-1]
+
+    def pop_meta(self, name):
+        """Pop metadata about the current statement from the metadata stack
+        for the current statement.
+
+        :name: The name of the metadata
+        """
+        self._dlog("getting meta '{}'".format(name))
+        return self._curr_scope["meta"][name].pop()
 
     def add_var(self, field_name, field, root=False):
         """Add a var to the current scope (vars are fields that
@@ -455,7 +483,6 @@ class Scope(object):
         :name: The name of the id
         :recurse: Whether parent scopes should also be searched (defaults to True)
         :returns: TODO
-
         """
         self._dlog("getting var '{}'".format(name))
         return self._search("vars", name, recurse)
@@ -853,7 +880,7 @@ class PfpInterp(object):
             res = None
             for child in ast.children():
                 res = self._handle_node(
-                    child, self._scope, self._ctxt, self._stream
+                    child, self._scope, self._ctxt, self._stream,
                 )
             return res
         except errors.InterpReturn as e:
@@ -1139,6 +1166,7 @@ class PfpInterp(object):
                     and not  is_forward_declared_struct(child):
                 continue
             self._handle_node(child, scope, ctxt, stream)
+            scope.clear_meta()
 
         for child in children:
             if type(child) is tuple:
@@ -1176,9 +1204,13 @@ class PfpInterp(object):
         """
         self._dlog("handling cast")
         to_type = self._handle_node(node.to_type, scope, ctxt, stream)
+
+        scope.push_meta("dest_type", to_type)
         val_to_cast = self._handle_node(node.expr, scope, ctxt, stream)
+        scope.pop_meta("dest_type")
 
         res = to_type()
+
         res._pfp__set_value(val_to_cast)
         return res
 
@@ -1584,6 +1616,7 @@ class PfpInterp(object):
             max_pos = 0
             for decl in node.decls:
                 self._handle_node(decl, scope, ctxt, stream)
+                scope.clear_meta()
 
         finally:
             # the union will have reset the stream
@@ -1666,6 +1699,7 @@ class PfpInterp(object):
             for decl in node.decls:
                 # new context! (struct)
                 self._handle_node(decl, scope, ctxt, stream)
+                scope.clear_meta()
 
             ctxt._pfp__process_fields_metadata()
 
@@ -1821,18 +1855,29 @@ class PfpInterp(object):
             "%": lambda x, y: x % y,
             ">": lambda x, y: x > y,
             "<": lambda x, y: x < y,
-            "||": lambda x, y: x or y,
+            "||": lambda x, y: 1 if x or y else 0,
             ">=": lambda x, y: x >= y,
             "<=": lambda x, y: x <= y,
             "==": lambda x, y: x == y,
             "!=": lambda x, y: x != y,
-            "&&": lambda x, y: x and y,
+            "&&": lambda x, y: 1 if x and y else 0,
             ">>": lambda x, y: x >> y,
             "<<": lambda x, y: x << y,
         }
 
+        dest_type = scope.get_meta("dest_type")
+
         left_val = self._handle_node(node.left, scope, ctxt, stream)
+        if dest_type is not None and not isinstance(left_val, dest_type):
+            new_left_val = dest_type()
+            new_left_val._pfp__set_value(left_val)
+            left_val = new_left_val
+
         right_val = self._handle_node(node.right, scope, ctxt, stream)
+        if dest_type is not None and not isinstance(right_val, dest_type):
+            new_right_val = dest_type()
+            new_right_val._pfp__set_value(right_val)
+            right_val = new_right_val
 
         if node.op not in switch:
             raise errors.UnsupportedBinaryOperator(node.coord, node.op)
@@ -2057,10 +2102,20 @@ class PfpInterp(object):
             "=": assign_op,
         }
 
+        scope.clear_meta()
+
         self._dlog("handling assignment")
         field = self._handle_node(node.lvalue, scope, ctxt, stream)
         self._dlog("field = {}".format(field))
-        value = self._handle_node(node.rvalue, scope, ctxt, stream)
+
+        scope.push_meta("dest_type", field._pfp__get_class())
+
+        value = self._handle_node(
+            node.rvalue,
+            scope,
+            ctxt,
+            stream,
+        )
 
         if node.op is None:
             self._dlog("value = {}".format(value))
@@ -2185,6 +2240,7 @@ class PfpInterp(object):
 
         try:
             for child in node.children():
+                scope.clear_meta()
                 self._handle_node(child, scope, ctxt, stream)
 
         # in case a return occurs, be sure to pop the scope
@@ -2398,7 +2454,7 @@ class PfpInterp(object):
                 except errors.InterpContinue as e:
                     pass
             if node.cond is not None and not self._handle_node(
-                node.cond, scope, ctxt, stream
+                node.cond, scope, ctxt, stream,
             ):
                 break
 
@@ -2594,7 +2650,6 @@ class PfpInterp(object):
         :returns: TODO
 
         """
-
         res = self._handle_node(node, scope, ctxt, stream)
 
         if isinstance(res, fields.Field):


Test patch:
diff --git a/tests/test_arrays.py b/tests/test_arrays.py
index 450995d..f98a224 100644
--- a/tests/test_arrays.py
+++ b/tests/test_arrays.py
@@ -287,7 +287,6 @@ class TestArrays(utils.PfpTestCase):
                     uint array[1];
                 } TestStruct;
             """,
-            verify=False,
         )
         struct = dom.TestStruct()
         struct.array = [0xFFFF]
diff --git a/tests/test_basic.py b/tests/test_basic.py
index d735e91..1739846 100644
--- a/tests/test_basic.py
+++ b/tests/test_basic.py
@@ -212,6 +212,180 @@ class TestBasic(utils.PfpTestCase):
             """,
         )
 
+    def test_add_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 2;
+                local int j = 3;
+                local int k = i + j;
+                Printf("%d", k);
+            """,
+            stdout="5",
+        )
+
+    def test_minus_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 2;
+                local int j = 3;
+                local int k = i - j;
+                Printf("%d", k);
+            """,
+            stdout="-1",
+        )
+
+    def test_mul_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 2;
+                local int j = 3;
+                local int k = i * j;
+                Printf("%d", k);
+            """,
+            stdout="6",
+        )
+
+    def test_div_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 2;
+                local int j = 6;
+                local int k = j / i;
+                Printf("%d", k);
+            """,
+            stdout="3",
+        )
+
+    def test_mod_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 5;
+                local int j = 7;
+                local int k = j % i;
+                Printf("%d", k);
+            """,
+            stdout="2",
+        )
+
+    def test_xor_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 6;
+                local int j = 3;
+                local int k = j ^ i;
+                Printf("%d", k);
+            """,
+            stdout="5",
+        )
+
+    def test_and_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 56; // 0b111000
+                local int j = 25; // 0b011001
+                local int k = j & i;
+                Printf("%d", k);
+            """,
+            stdout=str(int("011000", 2)),
+        )
+
+    def test_or_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 56; // 0b111000
+                local int j = 25; // 0b011001
+                local int k = j | i;
+                Printf("%d", k);
+            """,
+            stdout=str(int("111001", 2)),
+        )
+
+    def test_logical_or_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 0;
+                local int j = 25;
+                local int k = j || i;
+                Printf("%d", k);
+            """,
+            stdout="1",
+        )
+
+    def test_logical_or_operator2(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                Printf(
+                    "%d,%d,%d,%d",
+                    0 || 0,
+                    0 || 1,
+                    1 || 0,
+                    1 || 1
+                );
+            """,
+            stdout="0,1,1,1",
+        )
+
+    def test_logical_and_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 0;
+                local int j = 25;
+                local int k = j && i;
+                Printf("%d", k);
+            """,
+            stdout="0",
+        )
+
+    def test_logical_and_operator2(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                Printf(
+                    "%d,%d,%d,%d",
+                    0 && 0,
+                    0 && 1,
+                    1 && 0,
+                    1 && 1
+                );
+            """,
+            stdout="0,0,0,1",
+        )
+
+    def test_logical_shl_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 3;
+                local int j = 4;
+                local int k = i << j;
+                Printf("%d", k);
+            """,
+            stdout="48",
+        )
+
+    def test_logical_shr_operator(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local int i = 0x17000;
+                local int j = 4;
+                local int k = i >> j;
+                Printf("%d", k);
+            """,
+            stdout="5888",
+        )
+
     def test_local_accessible_via_this(self):
         dom = self._test_parse_build(
             "\x01\x02\x03\x04",
@@ -222,7 +396,6 @@ class TestBasic(utils.PfpTestCase):
                     Printf("%d", test);
                 } blah;
             """,
-            verify=False,
             stdout="{a},{a}".format(a=str(0x04030201)),
         )
 
@@ -240,7 +413,6 @@ class TestBasic(utils.PfpTestCase):
                     Printf("%d", test);
                 } blah;
             """,
-            verify=False,
             stdout="{a},{a}".format(a=str(0x04030201)),
         )
 
diff --git a/tests/test_cast.py b/tests/test_cast.py
index f9ba96e..7d4ee7d 100644
--- a/tests/test_cast.py
+++ b/tests/test_cast.py
@@ -34,6 +34,18 @@ class TestArrays(utils.PfpTestCase):
             stdout="a",
         )
 
+    def test_cast_from_dex(self):
+        dom = self._test_parse_build(
+            "",
+            """
+                local ubyte cur = 7;
+                local uint test1 = (uint)(10);
+                local uint test2 = (uint)((cur & 0x7f) << 7);
+                Printf("%u,%u", test1, test2);
+            """,
+            stdout="10,896",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_compat.py b/tests/test_compat.py
index d6b6245..c3c5f26 100644
--- a/tests/test_compat.py
+++ b/tests/test_compat.py
@@ -137,7 +137,6 @@ class TestCompatIO(utils.PfpTestCase):
                 Printf("%d|", blah);
                 Printf("%d", FTell());
             """,
-            verify=False,
             stdout="32769|0",
         )
 
@@ -153,7 +152,6 @@ class TestCompatIO(utils.PfpTestCase):
                 uchar b;
                 Printf("%d%d", a, b);
             """,
-            verify=False,
             stdout="ab9798",
         )
 
@@ -204,7 +202,6 @@ class TestCompatIO(utils.PfpTestCase):
                 Printf("%d", FSeek(FTell() + 4));
                 Printf("%d", FSeek(FTell() + 2));
             """,
-            verify=False,
             stdout="0-1",
         )
 
@@ -226,7 +223,6 @@ class TestCompatIO(utils.PfpTestCase):
                 local int rv6 = FSeek(0);
                 local int pos7 = FTell();
             """,
-            verify=False,
         )
         self.assertEqual(dom.pos1, 0)
         self.assertEqual(dom.rv1, 0)
@@ -293,7 +289,6 @@ class TestCompatIO(utils.PfpTestCase):
                 local int fskip_rv2 = FSkip(3);
                 local int pos2 = FTell();
             """,
-            verify=False,
         )
         self.assertEqual(dom.fskip_rv1, 0)
         self.assertEqual(dom.pos1, 1)
@@ -542,7 +537,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:3start-size:5-5start-size:17-5start-size:28-5",
-            verify=False,
             predefines=True,
         )
 
@@ -562,7 +556,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:3start-size:5-5start-size:17-5start-size:28-5",
-            verify=False,
             predefines=True,
         )
 
@@ -582,7 +575,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:3start-size:5-5start-size:17-5start-size:28-5",
-            verify=False,
             predefines=True,
         )
 
@@ -602,7 +594,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:3start-size:5-5start-size:22-5start-size:33-5",
-            verify=False,
             predefines=True,
         )
 
@@ -627,7 +618,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:3start-size:5-5start-size:22-5start-size:33-5",
-            verify=False,
             predefines=True,
         )
 
@@ -652,7 +642,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:3start-size:5-5start-size:16-5start-size:27-5",
-            verify=False,
             predefines=True,
         )
 
@@ -681,7 +670,6 @@ class TestCompatTools(utils.PfpTestCase):
                 }
             """,
             stdout="count:2start-size:5-5start-size:16-5",
-            verify=False,
             predefines=True,
         )
 
@@ -696,7 +684,6 @@ class TestCompatTools(utils.PfpTestCase):
                 index = FindNext();
                 Printf("%d", index);
             """,
-            verify=False,
             stdout="5,16,27",
         )
 
diff --git a/tests/test_functions.py b/tests/test_functions.py
index fb4b302..ad88ca9 100644
--- a/tests/test_functions.py
+++ b/tests/test_functions.py
@@ -155,7 +155,6 @@ class TestFunctions(utils.PfpTestCase):
                     Printf("false");
                 }
             """,
-            verify=False,
             stdout="true",
         )
 
diff --git a/tests/test_strings.py b/tests/test_strings.py
index 6bf4a0d..7ced832 100644
--- a/tests/test_strings.py
+++ b/tests/test_strings.py
@@ -121,7 +121,6 @@ class TestStrings(utils.PfpTestCase):
                              Printf("%s", temp_expected);
                      """,
             stdout="0102030405",
-            verify=False,
         )
 
     def test_add_strings_simple(self):
@@ -136,7 +135,6 @@ class TestStrings(utils.PfpTestCase):
                                  Printf("%s", test);
                          """,
             stdout="testtesttesttesttest",
-            verify=False,
         )
 
 
diff --git a/tests/utils.py b/tests/utils.py
index 4e8255e..13ef1ab 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -59,7 +59,6 @@ class PfpTestCase(unittest.TestCase):
         stdout=None,
         debug=False,
         predefines=False,
-        verify=True,
         _stream=True,
         printf=True,
     ):