Repository: angr/angr

Issue:
state.posix.open takes a long time to open large files
It takes 40 mins to open a file with size 128 MB.

```
import angr
import logging
from angr.state_plugins.posix import Flags

logger = logging.getLogger(name=__name__)
logger.setLevel(logging.INFO)

args = ["/bin/ls",
        "fs.img"]

p = angr.Project(args[0])
state = p.factory.blank_state(concrete_fs=True, mode="tracing")
state.options.remove(angr.sim_options.COPY_STATES)
logger.info("Opening file")
state.posix.open(args[1], Flags.O_RDWR)
logger.info("Done")
```

```
(pypy) ubuntu@thingy:~/$ python test_open.py 
INFO     | 2024-02-02 00:19:05,925 | __main__       | Opening file
INFO     | 2024-02-02 00:59:56,922 | __main__       | Done
```


Discussion:
@rhelmot Can we use mmap to solve this issue?
There are a couple of ways we can solve this problem. The way I was hoping would work was that breaking the object into 4k-sized BVVs and storing one per page instead of trying to create one BVV for the entire file would work, but apparently that's too slow. I was going to profile it and see if there was some other bottleneck. Barring that, the next approach would be to use SimMemory's backer feature to use the blob of the file as fallback data without having to have it as BVVs until it was accessed. I would rather not use mmap because we don't have a good way to reclaim that file descriptor other than just letting it be garbage collected.

Resolving patch:
diff --git a/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py b/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py
--- a/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py
+++ b/angr/storage/memory_mixins/paged_memory/pages/ultra_page.py
@@ -136,6 +136,8 @@ def store(
         if size >= memory.page_size - addr:
             size = memory.page_size - addr
 
+        byte_width = memory.state.arch.byte_width
+
         if type(data) is not int:
             if data.object.op == "BVV" and not data.object.annotations:
                 # trim the unnecessary leading bytes if there are any
@@ -143,14 +145,17 @@ def store(
                 start = (page_addr + addr - data.base) & ((1 << memory.state.arch.bits) - 1)
                 if start >= data.base + data.length:
                     raise SimMemoryError("Not enough bytes to store.")
-                start_bits = full_bits - start * memory.state.arch.byte_width - 1
+                start_bits = full_bits - start * byte_width - 1
                 # trim the overflowing bytes if there are any
-                end_bits = start_bits + 1 - size * memory.state.arch.byte_width
+                end_bits = start_bits + 1 - size * byte_width
                 if start_bits != full_bits - 1 or end_bits != 0:
                     if endness == "Iend_LE":
                         start_bits, end_bits = len(data.object) - end_bits - 1, len(data.object) - start_bits - 1
-                    obj = data.object[start_bits:end_bits]
-                    data = obj.args[0]
+
+                    concrete_data: bytes = data.concrete_bytes(
+                        (full_bits - start_bits - 1) // byte_width, (start_bits - end_bits + 1) // byte_width
+                    )
+                    data = int.from_bytes(concrete_data, "big")
 
         if type(data) is int or (data.object.op == "BVV" and not data.object.annotations):
             # mark range as not symbolic
diff --git a/angr/storage/memory_object.py b/angr/storage/memory_object.py
--- a/angr/storage/memory_object.py
+++ b/angr/storage/memory_object.py
@@ -1,4 +1,7 @@
+from typing import Optional
+
 import claripy
+
 from ..errors import SimMemoryError
 
 
@@ -24,6 +27,7 @@ class SimMemoryObject:
         "object",
         "length",
         "endness",
+        "_concrete_bytes",
     )
 
     def __init__(self, obj, base, endness, length=None, byte_width=8):
@@ -41,6 +45,7 @@ def __init__(self, obj, base, endness, length=None, byte_width=8):
         self.object = obj
         self.length = obj_bit_size(obj) // self._byte_width if length is None else length
         self.endness = endness
+        self._concrete_bytes: Optional[bytes] = None
 
     def size(self):
         return self.length * self._byte_width
@@ -64,6 +69,16 @@ def symbolic(self):
     def last_addr(self):
         return self.base + self.length - 1
 
+    def concrete_bytes(self, offset: int, size: int) -> Optional[bytes]:
+        if self._concrete_bytes is None:
+            if isinstance(self.object, claripy.ast.Bits) and self.object.op == "BVV" and not self.object.annotations:
+                self._concrete_bytes = self.object.concrete_value.to_bytes(len(self.object) // self._byte_width, "big")
+
+        if self._concrete_bytes is None:
+            return None
+
+        return self._concrete_bytes[offset : offset + size]
+
     def includes(self, x):
         return 0 <= x - self.base < self.length
 


Test patch: