Repository: eliben/pyelftools

Issue:
Supplemental DWARF vs DWARF elsewhere
Back when we were discussing #525, a question was posed - what is the difference between `.gnu_debuglink`, `.gnu_debugaltlink`, and `.debug_sup`. Back at the time, no good answer was given. Now, I've stumbled upon [this](https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html) - a description of how the GNU toolchain has the provision to store the debug info completely elsewhere, and the file with the debug info is linked to via the `.gnu_debuglink` section.

Meanwhile, supplementary DWARF is different; in that approach, the DWARF info (CUs/DIEs and so on) is present **in two files**, and the DWARF in the primary file contains explicit references to the DWARF in the supplementary file, via forms `DW_FORM_ref_supX`, `DW_FORM_strp_sup` etc. The supplementary file is linked via the `.debug_sup` section as of DWARF v5. Judging by #426, `.gnu_debugaltlink` is the pre-standard version of `.debug_sup`.

Point is, treating all three sections as semantically identical, the way #525 does, is wrong. If there is a `.gnu_debuglink` in a file, the logic should be to pull DWARF from the other file and return it as if it was primary, instead of sticking it into the supplementary reference.

Hints from issue comments:

Solution patch:
diff --git a/elftools/dwarf/datatype_cpp.py b/elftools/dwarf/datatype_cpp.py
index ee82f02..55e1ae0 100644
--- a/elftools/dwarf/datatype_cpp.py
+++ b/elftools/dwarf/datatype_cpp.py
@@ -12,7 +12,8 @@ from ..common.utils import bytes2str
 cpp_symbols = dict(
     pointer   = "*",
     reference = "&",
-    const     = "const")
+    const     = "const",
+    volatile  = "volatile")
 
 def describe_cpp_datatype(var_die):
     return str(parse_cpp_datatype(var_die))
@@ -36,8 +37,8 @@ def parse_cpp_datatype(var_die):
 
     mods = []
     # Unlike readelf, dwarfdump doesn't chase typedefs
-    while type_die.tag in ('DW_TAG_const_type', 'DW_TAG_pointer_type', 'DW_TAG_reference_type'):
-        modifier = _strip_type_tag(type_die) # const/reference/pointer
+    while type_die.tag in ('DW_TAG_const_type', 'DW_TAG_volatile_type', 'DW_TAG_pointer_type', 'DW_TAG_reference_type'):
+        modifier = _strip_type_tag(type_die) # const/volatile/reference/pointer
         mods.insert(0, modifier)
         if not 'DW_AT_type' in type_die.attributes: # void* is encoded as a pointer to nothing
             t.name = t.tag = "void"
@@ -145,9 +146,9 @@ class TypeDesc(object):
         mods = self.modifiers
 
         parts = []
-        # Initial const applies to the var ifself, other consts apply to the pointee
-        if len(mods) and mods[0] == 'const':
-            parts.append("const")
+        # Initial const/volatile applies to the var ifself, other consts apply to the pointee
+        if len(mods) and mods[0] in ('const', 'volatile'):
+            parts.append(mods[0])
             mods = mods[1:]
 
         # ref->const in the end, const goes in front
diff --git a/elftools/dwarf/dwarf_util.py b/elftools/dwarf/dwarf_util.py
index 811cc0b..6961572 100644
--- a/elftools/dwarf/dwarf_util.py
+++ b/elftools/dwarf/dwarf_util.py
@@ -7,7 +7,7 @@
 # This code is in the public domain
 #-------------------------------------------------------------------------------
 
-import os
+import os, binascii
 from ..construct.macros import UBInt32, UBInt64, ULInt32, ULInt64, Array
 from ..common.exceptions import DWARFError
 from ..common.utils import preserve_stream_pos, struct_parse
@@ -60,3 +60,14 @@ def _iter_CUs_in_section(stream, structs, parser):
         yield header
         offset = header.offset_after_length + header.unit_length   
 
+def _file_crc32(file):
+    """ Provided a readable binary stream, reads the stream to the end
+        and computes the CRC32 checksum of its contents,
+        with the initial value of 0.
+    """
+    d = file.read(4096)
+    checksum = 0
+    while len(d):
+        checksum = binascii.crc32(d, checksum)
+        d = file.read(4096)
+    return checksum
diff --git a/elftools/dwarf/dwarfinfo.py b/elftools/dwarf/dwarfinfo.py
index 71da941..2f3bebd 100644
--- a/elftools/dwarf/dwarfinfo.py
+++ b/elftools/dwarf/dwarfinfo.py
@@ -83,7 +83,6 @@ class DWARFInfo(object):
             debug_rnglists_sec,
             debug_sup_sec,
             gnu_debugaltlink_sec,
-            gnu_debuglink_sec,
             debug_types_sec
             ):
         """ config:
@@ -113,7 +112,6 @@ class DWARFInfo(object):
         self.debug_rnglists_sec = debug_rnglists_sec
         self.debug_sup_sec = debug_sup_sec
         self.gnu_debugaltlink_sec = gnu_debugaltlink_sec
-        self.gnu_debuglink_sec = gnu_debuglink_sec
         self.debug_types_sec = debug_types_sec
 
         # Sets the supplementary_dwarfinfo to None. Client code can set this
@@ -716,7 +714,7 @@ class DWARFInfo(object):
 
     def parse_debugsupinfo(self):
         """
-        Extract a filename from .debug_sup, .gnu_debualtlink sections, or .gnu_debuglink.
+        Extract a filename from .debug_sup, .gnu_debualtlink sections.
         """
         if self.debug_sup_sec is not None:
             self.debug_sup_sec.stream.seek(0)
@@ -727,9 +725,9 @@ class DWARFInfo(object):
             self.gnu_debugaltlink_sec.stream.seek(0)
             suplink = self.structs.Dwarf_debugaltlink.parse_stream(self.gnu_debugaltlink_sec.stream)
             return suplink.sup_filename
-        if self.gnu_debuglink_sec is not None:
-            self.gnu_debuglink_sec.stream.seek(0)
-            suplink = self.structs.Dwarf_debuglink.parse_stream(self.gnu_debuglink_sec.stream)
-            return suplink.sup_filename
+        # The section .gnu_debuglink with similarly looking contents
+        # has a different meaning - it doesn't point at supplementary DWARF,
+        # which is meant to be referenced from primary DWARF,
+        # it points at DWARF proper.
         return None
 
diff --git a/elftools/dwarf/structs.py b/elftools/dwarf/structs.py
index d3f6561..0f05c1b 100644
--- a/elftools/dwarf/structs.py
+++ b/elftools/dwarf/structs.py
@@ -177,7 +177,6 @@ class DWARFStructs(object):
 
         self._create_debugsup()
         self._create_gnu_debugaltlink()
-        self._create_gnu_debuglink()
 
     def _create_initial_length(self):
         def _InitialLength(name):
@@ -275,18 +274,6 @@ class DWARFStructs(object):
             CString("sup_filename"),
             String("sup_checksum", length=20))
 
-    def _create_gnu_debuglink(self):
-        self.Dwarf_debuglink = Struct('Elf_debuglink',
-            CString("sup_filename"),
-            Switch('', lambda ctx: (len(ctx.sup_filename) % 4),
-                {
-                    0: String("sup_padding", length=3),
-                    1: String("sup_padding", length=2),
-                    2: String("sup_padding", length=1),
-                    3: String("sup_padding", length=0),
-                }),
-            String("sup_checksum", length=4))
-
     def _create_dw_form(self):
         self.Dwarf_dw_form = dict(
             DW_FORM_addr=self.the_Dwarf_target_addr,
diff --git a/elftools/elf/elffile.py b/elftools/elf/elffile.py
index 034741b..e75bcf3 100644
--- a/elftools/elf/elffile.py
+++ b/elftools/elf/elffile.py
@@ -30,6 +30,7 @@ from ..dwarf.dwarfinfo import DWARFInfo, DebugSectionDescriptor, DwarfConfig
 from ..ehabi.ehabiinfo import EHABIInfo
 from .hash import ELFHashSection, GNUHashSection
 from .constants import SHN_INDICES
+from ..dwarf.dwarf_util import _file_crc32
 
 class ELFFile(object):
     """ Creation: the constructor accepts a stream (file-like object) with the
@@ -94,15 +95,24 @@ class ELFFile(object):
         ELFFile from it, setting up a correct stream_loader relative to the
         original file.
         """
-        base_directory = os.path.dirname(path)
-        def loader(elf_path):
-            # FIXME: use actual path instead of str/bytes
-            if not os.path.isabs(elf_path):
-                elf_path = os.path.join(base_directory,
-                                        elf_path)
-            return open(elf_path, 'rb')
         stream = open(path, 'rb')
-        return ELFFile(stream, loader)
+        return ELFFile(stream, ELFFile.make_relative_loader(path))
+    
+    @staticmethod
+    def make_relative_loader(base_path):
+        """ Return a function that takes a potentially relative path,
+            resolves it against base_path (bytes or str), and opens a file at that.
+
+            ELFFile uses functions like that for resolving DWARF links.
+        """
+        if isinstance(base_path, str):
+            base_path = base_path.encode('UTF-8') # resolver takes a bytes path
+        base_directory = os.path.dirname(base_path)
+        def loader(rel_path):
+            if not os.path.isabs(rel_path):
+                rel_path = os.path.join(base_directory, rel_path)
+            return open(rel_path, 'rb')
+        return loader
 
     def num_sections(self):
         """ Number of sections in the file
@@ -172,6 +182,13 @@ class ELFFile(object):
         if self._section_name_map is None:
             self._make_section_name_map()
         return self._section_name_map.get(section_name, None)
+    
+    def has_section(self, section_name):
+        """ Section existence check by name, without the overhead of parsing if found.
+        """
+        if self._section_name_map is None:
+            self._make_section_name_map()
+        return section_name in self._section_name_map
 
     def iter_sections(self, type=None):
         """ Yield all the sections in the file. If the optional |type|
@@ -231,14 +248,18 @@ class ELFFile(object):
                 end <= seg['p_vaddr'] + seg['p_filesz']):
                 yield start - seg['p_vaddr'] + seg['p_offset']
 
-    def has_dwarf_info(self):
+    def has_dwarf_info(self, strict=False):
         """ Check whether this file appears to have debugging information.
             We assume that if it has the .debug_info or .zdebug_info section, it
             has all the other required sections as well.
+
+            Unless you pass strict=True, the presence of .eh_frame section,
+            which is DWARF adjacent but hardly DWARF proper, will count as debug info.
+            Stripped files contain .eh_frame but none of the .[z]debug_xxx sections.
         """
-        return bool(self.get_section_by_name('.debug_info') or
-            self.get_section_by_name('.zdebug_info') or
-            self.get_section_by_name('.eh_frame'))
+        return (self.has_section('.debug_info') or
+            self.has_section('.zdebug_info') or
+            (not strict and self.has_section('.eh_frame')))
 
     def get_dwarf_info(self, relocate_dwarf_sections=True, follow_links=True):
         """ Return a DWARFInfo object representing the debugging information in
@@ -247,23 +268,39 @@ class ELFFile(object):
             If relocate_dwarf_sections is True, relocations for DWARF sections
             are looked up and applied.
 
-            If follow_links is True, we will try to load the supplementary
+            If follow_links is True, we will try to load the external and/or supplementary
             object file (if any), and use it to resolve references and imports.
         """
-        # Expect that has_dwarf_info was called, so at least .debug_info is
+        # Expect that has_dwarf_info() was called, so at least .debug_info is
         # present.
         # Sections that aren't found will be passed as None to DWARFInfo.
 
+        # TODO: support linking by build ID
+        # https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html
+
+        # A file may contain a debug link but not be stripped, so check for debug_info just in case
+        debuglink_section = self.get_section_by_name('.gnu_debuglink')
+        if debuglink_section and not self.has_dwarf_info(True) and follow_links and self.stream_loader:
+            debuglink = struct_parse(self.structs.Gnu_debuglink, debuglink_section.stream, debuglink_section.header.sh_offset)
+            with self.stream_loader(debuglink.filename) as ext_file:
+                # Validate checksum...
+                if _file_crc32(ext_file) != debuglink.checksum:
+                    raise ELFError('The linked DWARF file does not match the checksum in the link.')
+                ext_file.seek(0, os.SEEK_SET)
+                ext_elffile = ELFFile(ext_file, self.stream_loader)
+                # Inheriting the stream loader like that might be wrong if the supplementary DWARF link in the other file
+                # is relative to the other file's directory as opposed to this file's directory.
+                return ext_elffile.get_dwarf_info(relocate_dwarf_sections=relocate_dwarf_sections, follow_links=True)
+       
         section_names = ('.debug_info', '.debug_aranges', '.debug_abbrev',
                          '.debug_str', '.debug_line', '.debug_frame',
                          '.debug_loc', '.debug_ranges', '.debug_pubtypes',
                          '.debug_pubnames', '.debug_addr',
                          '.debug_str_offsets', '.debug_line_str',
                          '.debug_loclists', '.debug_rnglists',
-                         '.debug_sup', '.gnu_debugaltlink', '.gnu_debuglink',
-                         '.debug_types')
+                         '.debug_sup', '.gnu_debugaltlink', '.debug_types')
 
-        compressed = bool(self.get_section_by_name('.zdebug_info'))
+        compressed = self.has_section('.zdebug_info')
         if compressed:
             section_names = tuple(map(lambda x: '.z' + x[1:], section_names))
 
@@ -275,7 +312,7 @@ class ELFFile(object):
          debug_loc_sec_name, debug_ranges_sec_name, debug_pubtypes_name,
          debug_pubnames_name, debug_addr_name, debug_str_offsets_name,
          debug_line_str_name, debug_loclists_sec_name, debug_rnglists_sec_name,
-         debug_sup_name, gnu_debugaltlink_name, gnu_debuglink, debug_types_sec_name,
+         debug_sup_name, gnu_debugaltlink_name, debug_types_sec_name,
          eh_frame_sec_name) = section_names
 
         debug_sections = {}
@@ -318,13 +355,23 @@ class ELFFile(object):
                 debug_rnglists_sec=debug_sections[debug_rnglists_sec_name],
                 debug_sup_sec=debug_sections[debug_sup_name],
                 gnu_debugaltlink_sec=debug_sections[gnu_debugaltlink_name],
-                gnu_debuglink_sec=debug_sections[gnu_debuglink],
                 debug_types_sec=debug_sections[debug_types_sec_name]
                 )
         if follow_links:
             dwarfinfo.supplementary_dwarfinfo = self.get_supplementary_dwarfinfo(dwarfinfo)
         return dwarfinfo
-
+    
+    def has_dwarf_link(self):
+        """ Whether the binary's debug info is in an
+            external file. Use get_dwarf_link to retrieve the path to it.
+        """
+        return self.has_section('.gnu_debuglink')
+    
+    def get_dwarf_link(self):
+        """ Read the .gnu_debuglink section, return an object with filename (as bytes) and checksum (as number) in it.
+        """
+        section = self.get_section_by_name('.gnu_debuglink')
+        return struct_parse(self.structs.Gnu_debuglink, section.stream, section.header.sh_offset) if section else None
 
     def get_supplementary_dwarfinfo(self, dwarfinfo):
         """
diff --git a/elftools/elf/structs.py b/elftools/elf/structs.py
index 2af74aa..3a7ca66 100644
--- a/elftools/elf/structs.py
+++ b/elftools/elf/structs.py
@@ -112,6 +112,7 @@ class ELFStructs(object):
         self._create_riscv_attributes()
         self._create_elf_hash()
         self._create_gnu_hash()
+        self._create_gnu_debuglink()
 
     #-------------------------------- PRIVATE --------------------------------#
 
@@ -562,3 +563,9 @@ class ELFStructs(object):
                                self.Elf_word('bloom_shift'),
                                Array(lambda ctx: ctx['bloom_size'], self.Elf_xword('bloom')),
                                Array(lambda ctx: ctx['nbuckets'], self.Elf_word('buckets')))
+        
+    def _create_gnu_debuglink(self):
+        self.Gnu_debuglink = Struct('Gnu_debuglink',
+            CString("filename"),
+            Padding(lambda ctx: 3 - len(ctx.filename) % 4, strict=True),
+            self.Elf_word("checksum"))


Test patch:
diff --git a/test/test_debuglink.py b/test/test_debuglink.py
index b12baa6..932ea25 100644
--- a/test/test_debuglink.py
+++ b/test/test_debuglink.py
@@ -44,8 +44,8 @@ class TestDebuglink(unittest.TestCase):
         # Retrieve the subprograms from the DWARF info
         dwarf_info = elf.get_dwarf_info(follow_links=True, relocate_dwarf_sections=True)
 
-        if dwarf_info.supplementary_dwarfinfo:
-            for CU in dwarf_info.supplementary_dwarfinfo.iter_CUs():
+        if dwarf_info:
+            for CU in dwarf_info.iter_CUs():
                 for DIE in CU.iter_DIEs():
                     if DIE.tag == 'DW_TAG_subprogram':
                         attributes = DIE.attributes
@@ -70,9 +70,24 @@ class TestDebuglink(unittest.TestCase):
     def test_debuglink(self):
         with open('test/testfiles_for_unittests/debuglink', "rb") as elf_file:
             elf = ELFFile(elf_file, stream_loader=self.stream_loader)
+            # Contains eh_frame and gnu_debuglink, but no DWARF
+            self.assertTrue(elf.has_dwarf_info(False))
+            self.assertFalse(elf.has_dwarf_info(True))
+            self.assertTrue(elf.has_dwarf_link())
+
+            link = elf.get_dwarf_link()
+            self.assertIsNotNone(link)
+            self.assertEqual(link.filename, b'debuglink.debug')
+            self.assertEqual(link.checksum, 0x29b7c5f1)
+
             subprograms = self.subprograms_from_debuglink(elf)
             self.assertEqual(subprograms, {b'main': (0x1161, 0x52), b'addNumbers': (0x1149, 0x18)})
 
+        # Test the filesystem aware ELFFile loading
+        elf = ELFFile.load_from_path('test/testfiles_for_unittests/debuglink')
+        subprograms = self.subprograms_from_debuglink(elf)
+        self.assertEqual(subprograms, {b'main': (0x1161, 0x52), b'addNumbers': (0x1149, 0x18)})
+
 if __name__ == '__main__':
     unittest.main()
 
diff --git a/test/test_refaddr_bitness.py b/test/test_refaddr_bitness.py
index df05685..26ac508 100644
--- a/test/test_refaddr_bitness.py
+++ b/test/test_refaddr_bitness.py
@@ -51,7 +51,6 @@ class TestRefAddrOnDWARFv2With64BitTarget(unittest.TestCase):
             debug_rnglists_sec = None,
             debug_sup_sec = None,
             gnu_debugaltlink_sec = None,
-            gnu_debuglink_sec = None,
             debug_types_sec=None
         )