Repository: angr/angr

Issue:
Error when attempting to do VRA with workers=1
### Description

This occurs on some binaries when workers is set to 1. It appears to work fine for values > 1.
This is the backtrace that gets output:
```
Traceback (most recent call last):
  File "/usr/lib/python3.11/multiprocessing/queues.py", line 244, in _feed
    obj = _ForkingPickler.dumps(obj)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/multiprocessing/reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
RuntimeError: dictionary changed size during iteration
Traceback (most recent call last):
  File "/usr/lib/python3.11/multiprocessing/queues.py", line 244, in _feed
    obj = _ForkingPickler.dumps(obj)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/multiprocessing/reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
RuntimeError: dictionary changed size during iteration
```

### Steps to reproduce the bug

Folder with the reproduction script, binary source, and binary - [wrapper.zip](https://github.com/angr/angr/files/14661967/wrapper.zip)
Just run the test.py script

### Environment

_No response_

### Additional context

_No response_


Discussion:
This now appears to be fixed on master but not the pip release.
(pip release == 9.2.95)
(master == 9.2.96.dev0)
I don't think I did anything to fix this issue. I believe this issue still persists, but may not always be triggerable.
Hmmm, looks like you're right. Though I've never failed to trigger it on the pip version, the master version is intermittent.
I lied... I guess I'm just really lucky lol

Resolving patch:
diff --git a/angr/analyses/complete_calling_conventions.py b/angr/analyses/complete_calling_conventions.py
--- a/angr/analyses/complete_calling_conventions.py
+++ b/angr/analyses/complete_calling_conventions.py
@@ -88,11 +88,13 @@ def __init__(
         self._results = []
         if workers > 0:
             self._remaining_funcs = _mp_context.Value("i", 0)
-            self._func_queue = _mp_context.Queue()
             self._results = _mp_context.Queue()
+            self._results_lock = _mp_context.Lock()
+            self._func_queue = _mp_context.Queue()
             self._func_queue_lock = _mp_context.Lock()
         else:
             self._remaining_funcs = None  # not needed
+            self._results_lock = None  # not needed
             self._func_queue = None  # not needed
             self._func_queue_lock = threading.Lock()
 
@@ -218,8 +220,8 @@ def work(self):
             # spawn workers to perform the analysis
             with self._func_queue_lock:
                 procs = [
-                    _mp_context.Process(target=self._worker_routine, args=(Initializer.get(),), daemon=True)
-                    for _ in range(self._workers)
+                    _mp_context.Process(target=self._worker_routine, args=(worker_id, Initializer.get()), daemon=True)
+                    for worker_id in range(self._workers)
                 ]
                 for proc_idx, proc in enumerate(procs):
                     self._update_progress(0, text=f"Spawning worker {proc_idx}...")
@@ -231,7 +233,13 @@ def work(self):
             self._update_progress(0)
             idx = 0
             while idx < total_funcs:
-                func_addr, cc, proto, proto_libname, varman = self._results.get(True)
+                try:
+                    with self._results_lock:
+                        func_addr, cc, proto, proto_libname, varman = self._results.get(True, timeout=0.01)
+                except queue.Empty:
+                    time.sleep(0.1)
+                    continue
+
                 func = self.kb.functions.get_by_addr(func_addr)
                 if cc is not None or proto is not None:
                     func.calling_convention = cc
@@ -260,13 +268,14 @@ def work(self):
                         depends_on[dependent].discard(func_addr)
                         if not depends_on[dependent]:
                             callee_prototypes = self._get_callees_cc_prototypes(dependent)
-                            self._func_queue.put((dependent, callee_prototypes))
+                            with self._func_queue_lock:
+                                self._func_queue.put((dependent, callee_prototypes))
                             del depends_on[dependent]
 
             for proc in procs:
                 proc.join()
 
-    def _worker_routine(self, initializer: Initializer):
+    def _worker_routine(self, worker_id: int, initializer: Initializer):
         initializer.initialize()
         idx = 0
         while self._remaining_funcs.value > 0:
@@ -293,9 +302,10 @@ def _worker_routine(self, initializer: Initializer):
             try:
                 cc, proto, proto_libname, varman = self._analyze_core(func_addr)
             except Exception:  # pylint:disable=broad-except
-                _l.error("Exception occurred during _analyze_core().", exc_info=True)
+                _l.error("Worker %d: Exception occurred during _analyze_core().", worker_id, exc_info=True)
                 cc, proto, proto_libname, varman = None, None, None, None
-            self._results.put((func_addr, cc, proto, proto_libname, varman))
+            with self._results_lock:
+                self._results.put((func_addr, cc, proto, proto_libname, varman))
 
     def _analyze_core(
         self, func_addr: int
diff --git a/angr/knowledge_plugins/types.py b/angr/knowledge_plugins/types.py
--- a/angr/knowledge_plugins/types.py
+++ b/angr/knowledge_plugins/types.py
@@ -56,6 +56,12 @@ def __iter__(self):
         yield from super().__iter__()
         yield from iter(ALL_TYPES)
 
+    def __getstate__(self):
+        return dict(super().__iter__())  # do not pickle self.kb
+
+    def __setstate__(self, state):
+        self.update(state)
+
     def iter_own(self):
         """
         Iterate over all the names which are stored in this object - i.e. ``values()`` without ``ALL_TYPES``
diff --git a/angr/knowledge_plugins/variables/variable_manager.py b/angr/knowledge_plugins/variables/variable_manager.py
--- a/angr/knowledge_plugins/variables/variable_manager.py
+++ b/angr/knowledge_plugins/variables/variable_manager.py
@@ -122,7 +122,32 @@ def __setstate__(self, state):
         self.__dict__.update(state)
 
     def __getstate__(self):
-        d = dict(self.__dict__)
+        attributes = [
+            "func_addr",
+            "_variables",
+            "_global_region",
+            "_stack_region",
+            "_register_region",
+            "_live_variables",
+            "_variable_accesses",
+            "_insn_to_variable",
+            "_stmt_to_variable",
+            "_variable_to_stmt",
+            "_atom_to_variable",
+            "_ident_to_variable",
+            "_variable_counters",
+            "_unified_variables",
+            "_variables_to_unified_variables",
+            "_phi_variables",
+            "_variables_to_phivars",
+            "_phi_variables_by_block",
+            "types",
+            "variable_to_types",
+            "variables_with_manual_types",
+            "_variables_without_writes",
+            "ret_val_size",
+        ]
+        d = {k: getattr(self, k) for k in attributes}
         d["manager"] = None
         d["types"].kb = None
         return d
@@ -759,10 +784,11 @@ def has_read_access(accesses):
             if variable in self._phi_variables:
                 # a phi variable is definitely not an input variable
                 continue
-            accesses = self._variable_accesses[variable]
-            if has_read_access(accesses):
-                if not exclude_specials or not variable.category:
-                    input_variables.append(variable)
+            if variable in self._variable_accesses:
+                accesses = self._variable_accesses[variable]
+                if has_read_access(accesses):
+                    if not exclude_specials or not variable.category:
+                        input_variables.append(variable)
 
         return input_variables
 


Test patch: