Repository: angr/angr

Issue:
VFG is generated without any edges
@jackaaron and I figured out was that `VFGJob` was being created without a source block ID.  The source block ID seems to be used to generate the edges between a node and its predecessor by `vfg._graph_add_edge`.  If a VFGJob is created without a `src_block_id`, then no edges are created.

With the below changes, we were able to get a reasonable VFG with edges.

```diff
diff --git angr/analyses/vfg.py angr/analyses/vfg.py
index 9af3bcc..508e845 100644
--- angr/analyses/vfg.py
+++ angr/analyses/vfg.py
@@ -404,7 +404,8 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
                 return n
 
     def irsb_from_node(self, node):
-        return self.project.factory.successors(node.state, addr=node.addr, num_inst=len(node.instruction_addrs))
+         num_inst = len(self.project.factory.block(addr=node.addr).instruction_addrs)
+         return self.project.factory.successors(node.state, addr=node.addr, num_inst=num_inst)
 
     def get_paths(self, begin, end):
         """
@@ -415,6 +416,7 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
         a_paths = []
         for p in paths:
             runs = map(self.irsb_from_node, p)
+            raise DeprecationWarning("angr.path.make_path() is no longer")
             a_paths.append(angr.path.make_path(self.project, runs))
         return a_paths
 
@@ -989,7 +991,8 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
         merged_state, _ = self._merge_states(state_0, state_1)
 
         new_job = VFGJob(jobs[0].addr, merged_state, self._context_sensitivity_level, jumpkind=jobs[0].jumpkind,
-                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack
+                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack,
+                         src_block_id=[jobs[0].src_block_id, jobs[1].src_block_id] # XXX: something bad might happen
                          )
 
         self._top_function_analysis_task.jobs.append(new_job)
@@ -1040,7 +1043,8 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
         # import ipdb; ipdb.set_trace()
 
         new_job = VFGJob(jobs[0].addr, new_state, self._context_sensitivity_level, jumpkind=jobs[0].jumpkind,
-                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack
+                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack,
+                         src_block_id=[jobs[0].src_block_id, jobs[1].src_block_id] # XXX: something bad might happen
                          )
         self._top_function_analysis_task.jobs.append(new_job)
 
@@ -1316,8 +1320,10 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
             self.graph.add_node(dst_node)
 
         else:
-            src_node = self._graph_get_node(src_block_id, terminator_for_nonexistent_node=True)
-            self.graph.add_edge(src_node, dst_node, **kwargs)
+            src_block_id = [src_block_id] if not isinstance(src_block_id,list) else src_block_id
+            for src_blk_id in src_block_id:
+                src_node = self._graph_get_node(src_blk_id, terminator_for_nonexistent_node=True)
+                self.graph.add_edge(src_node, dst_node, **kwargs)
 
     #
     # Other methods
@@ -1428,6 +1434,7 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
                                  block_id=new_block_id,
                                  jumpkind='Ijk_Ret',
                                  call_stack=new_call_stack,
+                                 src_block_id=job._block_id
                                  )
 
                 new_jobs.append(new_job)
@@ -1473,6 +1480,7 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
                              block_id=new_block_id,
                              jumpkind=successor_state.history.jumpkind,
                              call_stack=new_call_stack,
+                             src_block_id=job._block_id
                              )
 
             if successor.history.jumpkind == 'Ijk_Ret':
@@ -1711,6 +1719,7 @@ class VFG(ForwardAnalysis, Analysis):   # pylint:disable=abstract-method
                      block_id=block_id,
                      jumpkind=state.history.jumpkind,
                      call_stack=call_stack,
+                     src_block_id=job_key._block_id
                      )
         self._insert_job(job)
```

We thought that by getting a graph with nodes containing successors that we would be able to get a VSA DDG with nodes, but alas that was not to be.  We ran into a situation similar to what was asked in https://lists.cs.ucsb.edu/pipermail/angr/2018-January/000427.html.  Either our mods weren't right or there might be some other issues.


Discussion:
I think this is also related to https://github.com/angr/angr/issues/1038#issuecomment-396545863
Using a unit test binary (vfg_0), the following script shows that there are no edges in the VFG.

```python
#!/usr/bin/env python                                                                                

import angr,os,logging

def main():
    fname = os.path.join(os.getcwd(),"../binaries/tests/x86_64/vfg_0")
    proj = angr.Project(fname,load_options={'auto_load_libs': False})

    cfg = proj.analyses.CFGAccurate(normalize=True)
    main = cfg.functions.function(name='main')
    logging.getLogger('angr.analyses.vfg').setLevel(logging.DEBUG)
    vfg = proj.analyses.VFG(cfg, \
                            start=main.addr, \
                            context_sensitivity_level=1, \
                            interfunction_level=3, \
                            record_function_final_states=True, \
                            max_iterations=80, \
                            )

    print "VFG num edges : {}".format(vfg.graph.number_of_edges())
    print "VFG num nodes : {}".format(vfg.graph.number_of_nodes())

    import pdb
    pdb.set_trace()

if __name__ == "__main__":
    main()
```

```
VFG num edges : 0
VFG num nodes : 7
```
Just as an update: I plan to work on this (and a few other VFG-related issues) after finishing up my current refactor of CFGFast.
I'm reading https://github.com/0day1day/bap/blob/master/ocaml/vsa.ml, https://github.com/0day1day/bap/blob/master/ocaml/vsa_ast.ml and https://github.com/0day1day/bap/blob/master/ocaml/vsa_ssa.ml and also
skimming https://research.cs.wisc.edu/wpis/papers/balakrishnan_thesis.pdf to try to figure out how I can fix it up correctly myself.

Separately, I guess the angr-doc "Widening" section on explaining what it means is basically answered in Section 7.1 (and 7.3) of the thesis.  I haven't looked enough at the angr code to determine which section is most applicable.

Resolving patch:
diff --git a/angr/analyses/vfg.py b/angr/analyses/vfg.py
--- a/angr/analyses/vfg.py
+++ b/angr/analyses/vfg.py
@@ -65,6 +65,19 @@ def callstack_repr(self, kb=None):
         return "//".join(s)
 
 
+class PendingJob(object):
+
+    __slots__ = ('block_id', 'state', 'call_stack', 'src_block_id', 'src_stmt_idx', 'src_ins_addr', )
+
+    def __init__(self, block_id, state, call_stack, src_block_id, src_stmt_idx, src_ins_addr):
+        self.block_id = block_id
+        self.state = state
+        self.call_stack = call_stack
+        self.src_block_id = src_block_id
+        self.src_stmt_idx = src_stmt_idx
+        self.src_ins_addr = src_ins_addr
+
+
 class AnalysisTask(object):
     """
     An analysis task describes a task that should be done before popping this task out of the task stack and discard it.
@@ -404,19 +417,7 @@ def get_any_node(self, addr):
                 return n
 
     def irsb_from_node(self, node):
-        return self.project.factory.successors(node.state, addr=node.addr, num_inst=len(node.instruction_addrs))
-
-    def get_paths(self, begin, end):
-        """
-        Get all the simple paths between @begin and @end.
-        Returns: a list of angr.Path instances.
-        """
-        paths = self._get_nx_paths(begin, end)
-        a_paths = []
-        for p in paths:
-            runs = map(self.irsb_from_node, p)
-            a_paths.append(angr.path.make_path(self.project, runs))
-        return a_paths
+        return self.project.factory.successors(node.state, addr=node.addr)
 
     #
     # Operations
@@ -687,7 +688,7 @@ def _get_successors(self, job):
         if job.sim_successors.sort == 'IRSB' and state.thumb:
             self._thumb_addrs.update(job.sim_successors.artifacts['insn_addrs'])
 
-        if len(all_successors) == 0:
+        if not all_successors:
             if job.sim_successors.sort == 'SimProcedure' and isinstance(job.sim_successors.artifacts['procedure'],
                     SIM_PROCEDURES["stubs"]["PathTerminator"]):
                 # If there is no valid exit in this branch and it's not
@@ -844,7 +845,7 @@ def _handle_successor(self, job, successor, all_successors):
             if new_block_id in self._pending_returns:
                 del self._pending_returns[new_block_id]
 
-        # Check if we have reached a fixpoint
+        # Check if we have reached a fix-point
         if jumpkind != 'Ijk_FakeRet' and \
                 new_block_id in self._nodes:
             last_state = self._nodes[new_block_id].state
@@ -855,6 +856,7 @@ def _handle_successor(self, job, successor, all_successors):
                 l.debug("%s didn't reach a fix-point", new_block_id)
             else:
                 l.debug("%s reaches a fix-point.", new_block_id)
+                job.dbg_exit_status[successor] = "Merged due to reaching a fix-point"
                 return [ ]
 
         new_jobs = self._create_new_jobs(job, successor, new_block_id, new_call_stack)
@@ -989,7 +991,8 @@ def _merge_jobs(self, *jobs):
         merged_state, _ = self._merge_states(state_0, state_1)
 
         new_job = VFGJob(jobs[0].addr, merged_state, self._context_sensitivity_level, jumpkind=jobs[0].jumpkind,
-                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack
+                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack, src_block_id=jobs[0].src_block_id,
+                         src_exit_stmt_idx=jobs[0].src_exit_stmt_idx, src_ins_addr=jobs[0].src_ins_addr,
                          )
 
         self._top_function_analysis_task.jobs.append(new_job)
@@ -1040,7 +1043,8 @@ def _widen_jobs(self, *jobs):
         # import ipdb; ipdb.set_trace()
 
         new_job = VFGJob(jobs[0].addr, new_state, self._context_sensitivity_level, jumpkind=jobs[0].jumpkind,
-                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack
+                         block_id=jobs[0].block_id, call_stack=jobs[0].call_stack, src_block_id=jobs[0].src_block_id,
+                         src_exit_stmt_idx=jobs[0].src_exit_stmt_idx, src_ins_addr=jobs[0].src_ins_addr,
                          )
         self._top_function_analysis_task.jobs.append(new_job)
 
@@ -1385,6 +1389,8 @@ def _create_new_jobs(self, job, successor, new_block_id, new_call_stack):
         # TODO: basic block stack is probably useless
 
         jumpkind = successor.history.jumpkind
+        stmt_idx = successor.scratch.stmt_idx
+        ins_addr = successor.scratch.ins_addr
         # Make a copy of the state in case we use it later
         successor_state = successor.copy()
         successor_addr = successor_state.se.eval(successor_state.ip)
@@ -1428,6 +1434,9 @@ def _create_new_jobs(self, job, successor, new_block_id, new_call_stack):
                                  block_id=new_block_id,
                                  jumpkind='Ijk_Ret',
                                  call_stack=new_call_stack,
+                                 src_block_id=job.block_id,
+                                 src_exit_stmt_idx=stmt_idx,
+                                 src_ins_addr=ins_addr,
                                  )
 
                 new_jobs.append(new_job)
@@ -1436,8 +1445,8 @@ def _create_new_jobs(self, job, successor, new_block_id, new_call_stack):
                 job.dbg_exit_status[successor] = "Pending"
 
             else:
-                self._pending_returns[new_block_id] = \
-                    (successor_state, new_call_stack)
+                self._pending_returns[new_block_id] = PendingJob(new_block_id, successor_state, new_call_stack,
+                                                                 job.block_id, stmt_idx, ins_addr)
                 job.dbg_exit_status[successor] = "Pending"
 
         else:
@@ -1473,6 +1482,9 @@ def _create_new_jobs(self, job, successor, new_block_id, new_call_stack):
                              block_id=new_block_id,
                              jumpkind=successor_state.history.jumpkind,
                              call_stack=new_call_stack,
+                             src_block_id=job.block_id,
+                             src_exit_stmt_idx=stmt_idx,
+                             src_ins_addr=ins_addr,
                              )
 
             if successor.history.jumpkind == 'Ijk_Ret':
@@ -1567,7 +1579,6 @@ def _post_job_handling_debug(self, job, successors):
         l.debug("-  is call jump: %s", job.is_call_jump)
         for suc in successors:
             if suc not in job.dbg_exit_status:
-                # TODO:
                 l.warning("- %s is not found. FIND OUT WHY.", suc)
                 continue
 
@@ -1695,22 +1706,25 @@ def _save_function_final_state(self, function_key, function_address, state):
 
     def _trace_pending_job(self, job_key):
 
-        state, call_stack = self._pending_returns.pop(job_key)
+        pending_job = self._pending_returns.pop(job_key)  # type: PendingJob
         addr = job_key.addr
 
         # Unlike CFG, we will still trace those blocks that have been traced before. In other words, we don't
-        # remove fake returns even if they have been traced - otherwise we cannot come to a fixpoint.
+        # remove fake returns even if they have been traced - otherwise we cannot come to a fix-point.
 
         block_id = BlockID.new(addr,
-                               call_stack.stack_suffix(self._context_sensitivity_level),
+                               pending_job.call_stack.stack_suffix(self._context_sensitivity_level),
                                'Ijk_Ret'
                                )
         job = VFGJob(addr,
-                     state,
+                     pending_job.state,
                      self._context_sensitivity_level,
                      block_id=block_id,
-                     jumpkind=state.history.jumpkind,
-                     call_stack=call_stack,
+                     jumpkind=pending_job.state.history.jumpkind,
+                     call_stack=pending_job.call_stack,
+                     src_block_id=pending_job.src_block_id,
+                     src_exit_stmt_idx=pending_job.src_stmt_idx,
+                     src_ins_addr=pending_job.src_ins_addr,
                      )
         self._insert_job(job)
         self._top_task.jobs.append(job)
@@ -1718,7 +1732,7 @@ def _trace_pending_job(self, job_key):
     def _get_pending_job(self, func_addr):
 
         pending_ret_key = None
-        for k in self._pending_returns.iterkeys():  # type: BlockID
+        for k in self._pending_returns.keys():  # type: BlockID
             if k.func_addr == func_addr:
                 pending_ret_key = k
                 break


Test patch:
diff --git a/tests/test_vfg_path.py b/tests/test_vfg_path.py
deleted file mode 100644
--- a/tests/test_vfg_path.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import angr
-import logging
-import os
-
-l = logging.getLogger("angr_tests")
-test_location = str(os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                 '../../binaries/tests'))
-
-def test_vfg_paths():
-    p = angr.Project(os.path.join(test_location, "x86_64/track_user_input"))
-    main_addr = p.loader.find_symbol("main").rebased_addr
-    printf_addr = 0x4005e1 # actually where it returns
-
-    vfg = p.analyses.VFG(context_sensitivity_level=1, interfunction_level=5)
-    paths = vfg.get_paths(main_addr, printf_addr)
-
-if __name__ == '__main__':
-    test_vfg_paths()