Repository: vivisect/vivisect

Issue:
i386 rep emulation incorrect
The current emulator runs the rep prefix for every instruction.

Per the Intel manual: "The behavior of the REP prefix is undefined when used with non-string instructions."

Example assembly resulting in incorrect / slow emulation decrementing undefined `ecx` value.
```asm
.text:00402107 74 07                             jz      short loc_402110
.text:00402109 F3 C3                             rep retn
```

References:
- https://repzret.org/p/repzret/
- https://stackoverflow.com/questions/10258918/what-happens-when-a-rep-prefix-is-attached-to-a-non-string-instruction

Discussion:

Patch:
diff --git a/envi/archs/i386/emu.py b/envi/archs/i386/emu.py
index 658a6f7..02e9b6a 100755
--- a/envi/archs/i386/emu.py
+++ b/envi/archs/i386/emu.py
@@ -9,7 +9,7 @@ from envi.const import *
 import envi.exc as e_exc
 import envi.bits as e_bits
 
-from envi.archs.i386.opconst import PREFIX_REX_W
+from envi.archs.i386.opconst import PREFIX_REX_W, REP_OPCODES
 from envi.archs.i386.regs import *
 from envi.archs.i386.disasm import *
 from envi.archs.i386 import i386Module
@@ -245,8 +245,9 @@ class IntelEmulator(i386RegisterContext, envi.Emulator):
         if meth is None:
             raise e_exc.UnsupportedInstruction(self, op)
 
+        # The behavior of the REP prefix is undefined when used with non-string instructions.
         rep_prefix = op.prefixes & PREFIX_REP_MASK
-        if rep_prefix and not self.getEmuOpt('i386:reponce'):
+        if rep_prefix and op.opcode in REP_OPCODES and not self.getEmuOpt('i386:reponce'):
             # REP instructions (REP/REPNZ/REPZ/REPSIMD) get their own handlers
             handler = self.__rep_prefix_handlers__.get(rep_prefix)
             newpc = handler(meth, op)
@@ -273,9 +274,6 @@ class IntelEmulator(i386RegisterContext, envi.Emulator):
         Then the instruction is repeated and ECX decremented until either
         ECX reaches 0 or the ZF is cleared.
         '''
-        if op.mnem.startswith('nop'):
-            return
-
         ecx = emu.getRegister(REG_ECX)
         emu.setFlag(EFLAGS_ZF, 1)
 
diff --git a/envi/archs/i386/opconst.py b/envi/archs/i386/opconst.py
index 9fc2db9..4f9add5 100644
--- a/envi/archs/i386/opconst.py
+++ b/envi/archs/i386/opconst.py
@@ -197,15 +197,24 @@ INS_OFLOW = INS_TRAPS | 0x08       # gen overflow trap
 
 #/* INS_SYSTEM */
 INS_HALT    = INS_SYSTEM | 0x01 # halt machine
-INS_IN      = INS_SYSTEM | 0x02 # input form port
+INS_IN      = INS_SYSTEM | 0x02 # input from port
 INS_OUT     = INS_SYSTEM | 0x03 # output to port
 INS_CPUID   = INS_SYSTEM | 0x04 # iden
 
 INS_NOP     = INS_OTHER | 0x01
 INS_BCDCONV = INS_OTHER | 0x02  # convert to/from BCD
 INS_SZCONV  = INS_OTHER | 0x03  # convert size of operand
-INS_CRYPT   = INS_OTHER | 0x4  # AES-NI instruction support
+INS_CRYPT   = INS_OTHER | 0x04  # AES-NI instruction support
 
+# string instructions that support REP prefix
+REP_OPCODES = (
+    INS_IN,       # INS
+    INS_OUT,      # OUTS
+    INS_STRMOV,   # MOVS
+    INS_STRLOAD,  # LODS
+    INS_STRSTOR,  # STOS
+    INS_STRCMP    # CMPS, SCAS
+    )
 
 OP_R = 0x001
 OP_W = 0x002


Test patch:
diff --git a/envi/tests/test_arch_i386_emu.py b/envi/tests/test_arch_i386_emu.py
index 273a29d..d8b26e7 100644
--- a/envi/tests/test_arch_i386_emu.py
+++ b/envi/tests/test_arch_i386_emu.py
@@ -108,7 +108,12 @@ i386Tests = [
     # lea ecx,dword [esp + -973695896]  (test SIB getOperAddr)
     {'bytes': '8d8c246894f6c5',
      'setup': ({'esp': 0x7fd0}, {}),
-     'tests': ({'ecx': 0xc5f71438}, {})}
+     'tests': ({'ecx': 0xc5f71438}, {})},
+    # rep ret
+    # The behavior of the REP prefix is undefined when used with non-string instructions.
+    {'bytes': 'f3c3',
+     'setup': ({'ecx': 0x1}, {'esp': b'\x00\x00\x00\x60'}),
+     'tests': ({'ecx': 0x1, 'eip': 0x60000000}, {})}
 ]