Repository: eliben/pyelftools

Issue:
Supplemental DWARF vs DWARF elsewhere
Back when we were discussing #525, a question was posed - what is the difference between `.gnu_debuglink`, `.gnu_debugaltlink`, and `.debug_sup`. Back at the time, no good answer was given. Now, I've stumbled upon [this](https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html) - a description of how the GNU toolchain has the provision to store the debug info completely elsewhere, and the file with the debug info is linked to via the `.gnu_debuglink` section.

Meanwhile, supplementary DWARF is different; in that approach, the DWARF info (CUs/DIEs and so on) is present **in two files**, and the DWARF in the primary file contains explicit references to the DWARF in the supplementary file, via forms `DW_FORM_ref_supX`, `DW_FORM_strp_sup` etc. The supplementary file is linked via the `.debug_sup` section as of DWARF v5. Judging by #426, `.gnu_debugaltlink` is the pre-standard version of `.debug_sup`.

Point is, treating all three sections as semantically identical, the way #525 does, is wrong. If there is a `.gnu_debuglink` in a file, the logic should be to pull DWARF from the other file and return it as if it was primary, instead of sticking it into the supplementary reference.

Discussion:

Solution patch:
diff --git a/elftools/dwarf/datatype_cpp.py b/elftools/dwarf/datatype_cpp.py
index ee82f02..55e1ae0 100644
--- a/elftools/dwarf/datatype_cpp.py
+++ b/elftools/dwarf/datatype_cpp.py
@@ -12,7 +12,8 @@ from ..common.utils import bytes2str
 cpp_symbols = dict(
     pointer   = "*",
     reference = "&",
-    const     = "const")
+    const     = "const",
+    volatile  = "volatile")
 
 def describe_cpp_datatype(var_die):
     return str(parse_cpp_datatype(var_die))
@@ -36,8 +37,8 @@ def parse_cpp_datatype(var_die):
 
     mods = []
     # Unlike readelf, dwarfdump doesn't chase typedefs
-    while type_die.tag in ('DW_TAG_const_type', 'DW_TAG_pointer_type', 'DW_TAG_reference_type'):
-        modifier = _strip_type_tag(type_die) # const/reference/pointer
+    while type_die.tag in ('DW_TAG_const_type', 'DW_TAG_volatile_type', 'DW_TAG_pointer_type', 'DW_TAG_reference_type'):
+        modifier = _strip_type_tag(type_die) # const/volatile/reference/pointer
         mods.insert(0, modifier)
         if not 'DW_AT_type' in type_die.attributes: # void* is encoded as a pointer to nothing
             t.name = t.tag = "void"
@@ -145,9 +146,9 @@ class TypeDesc(object):
         mods = self.modifiers
 
         parts = []
-        # Initial const applies to the var ifself, other consts apply to the pointee
-        if len(mods) and mods[0] == 'const':
-            parts.append("const")
+        # Initial const/volatile applies to the var ifself, other consts apply to the pointee
+        if len(mods) and mods[0] in ('const', 'volatile'):
+            parts.append(mods[0])
             mods = mods[1:]
 
         # ref->const in the end, const goes in front
diff --git a/elftools/dwarf/dwarf_util.py b/elftools/dwarf/dwarf_util.py
index 811cc0b..6961572 100644
--- a/elftools/dwarf/dwarf_util.py
+++ b/elftools/dwarf/dwarf_util.py
@@ -7,7 +7,7 @@
 # This code is in the public domain
 #-------------------------------------------------------------------------------
 
-import os
+import os, binascii
 from ..construct.macros import UBInt32, UBInt64, ULInt32, ULInt64, Array
 from ..common.exceptions import DWARFError
 from ..common.utils import preserve_stream_pos, struct_parse
@@ -60,3 +60,14 @@ def _iter_CUs_in_section(stream, structs, parser):
         yield header
         offset = header.offset_after_length + header.unit_length   
 
+def _file_crc32(file):
+    """ Provided a readable binary stream, reads the stream to the end
+        and computes the CRC32 checksum of its contents,
+        with the initial value of 0.
+    """
+    d = file.read(4096)
+    checksum = 0
+    while len(d):
+        checksum = binascii.crc32(d, checksum)
+        d = file.read(4096)
+    return checksum
diff --git a/elftools/dwarf/dwarfinfo.py b/elftools/dwarf/dwarfinfo.py
index 71da941..2f3bebd 100644
--- a/elftools/dwarf/dwarfinfo.py
+++ b/elftools/dwarf/dwarfinfo.py
@@ -83,7 +83,6 @@ class DWARFInfo(object):
             debug_rnglists_sec,
             debug_sup_sec,
             gnu_debugaltlink_sec,
-            gnu_debuglink_sec,
             debug_types_sec
             ):
         """ config:
@@ -113,7 +112,6 @@ class DWARFInfo(object):
         self.debug_rnglists_sec = debug_rnglists_sec
         self.debug_sup_sec = debug_sup_sec
         self.gnu_debugaltlink_sec = gnu_debugaltlink_sec
-        self.gnu_debuglink_sec = gnu_debuglink_sec
         self.debug_types_sec = debug_types_sec
 
         # Sets the supplementary_dwarfinfo to None. Client code can set this
@@ -716,7 +714,7 @@ class DWARFInfo(object):
 
     def parse_debugsupinfo(self):
         """
-        Extract a filename from .debug_sup, .gnu_debualtlink sections, or .gnu_debuglink.
+        Extract a filename from .debug_sup, .gnu_debualtlink sections.
         """
         if self.debug_sup_sec is not None:
             self.debug_sup_sec.stream.seek(0)
@@ -727,9 +725,9 @@ class DWARFInfo(object):
             self.gnu_debugaltlink_sec.stream.seek(0)
             suplink = self.structs.Dwarf_debugaltlink.parse_stream(self.gnu_debugaltlink_sec.stream)
             return suplink.sup_filename
-        if self.gnu_debuglink_sec is not None:
-            self.gnu_debuglink_sec.stream.seek(0)
-            suplink = self.structs.Dwarf_debuglink.parse_stream(self.gnu_debuglink_sec.stream)
-            return suplink.sup_filename
+        # The section .gnu_debuglink with similarly looking contents
+        # has a different meaning - it doesn't point at supplementary DWARF,
+        # which is meant to be referenced from primary DWARF,
+        # it points at DWARF proper.
         return None
 
diff --git a/elftools/dwarf/structs.py b/elftools/dwarf/structs.py
index d3f6561..0f05c1b 100644
--- a/elftools/dwarf/structs.py
+++ b/elftools/dwarf/structs.py
@@ -177,7 +177,6 @@ class DWARFStructs(object):
 
         self._create_debugsup()
         self._create_gnu_debugaltlink()
-        self._create_gnu_debuglink()
 
     def _create_initial_length(self):
         def _InitialLength(name):
@@ -275,18 +274,6 @@ class DWARFStructs(object):
             CString("sup_filename"),
             String("sup_checksum", length=20))
 
-    def _create_gnu_debuglink(self):
-        self.Dwarf_debuglink = Struct('Elf_debuglink',
-            CString("sup_filename"),
-            Switch('', lambda ctx: (len(ctx.sup_filename) % 4),
-                {
-                    0: String("sup_padding", length=3),
-                    1: String("sup_padding", length=2),
-                    2: String("sup_padding", length=1),
-                    3: String("sup_padding", length=0),
-                }),
-            String("sup_checksum", length=4))
-
     def _create_dw_form(self):
         self.Dwarf_dw_form = dict(
             DW_FORM_addr=self.the_Dwarf_target_addr,
diff --git a/elftools/elf/elffile.py b/elftools/elf/elffile.py
index 034741b..e75bcf3 100644
--- a/elftools/elf/elffile.py
+++ b/elftools/elf/elffile.py
@@ -30,6 +30,7 @@ from ..dwarf.dwarfinfo import DWARFInfo, DebugSectionDescriptor, DwarfConfig
 from ..ehabi.ehabiinfo import EHABIInfo
 from .hash import ELFHashSection, GNUHashSection
 from .constants import SHN_INDICES
+from ..dwarf.dwarf_util import _file_crc32
 
 class ELFFile(object):
     """ Creation: the constructor accepts a stream (file-like object) with the
@@ -94,15 +95,24 @@ class ELFFile(object):
         ELFFile from it, setting up a correct stream_loader relative to the
         original file.
         """
-        base_directory = os.path.dirname(path)
-        def loader(elf_path):
-            # FIXME: use actual path instead of str/bytes
-            if not os.path.isabs(elf_path):
-                elf_path = os.path.join(base_directory,
-                                        elf_path)
-            return open(elf_path, 'rb')
         stream = open(path, 'rb')
-        return ELFFile(stream, loader)
+        return ELFFile(stream, ELFFile.make_relative_loader(path))
+    
+    @staticmethod
+    def make_relative_loader(base_path):
+        """ Return a function that takes a potentially relative path,
+            resolves it against base_path (bytes or str), and opens a file at that.
+
+            ELFFile uses functions like that for resolving DWARF links.
+        """
+        if isinstance(base_path, str):
+            base_path = base_path.encode('UTF-8') # resolver takes a bytes path
+        base_directory = os.path.dirname(base_path)
+        def loader(rel_path):
+            if not os.path.isabs(rel_path):
+                rel_path = os.path.join(base_directory, rel_path)
+            return open(rel_path, 'rb')
+        return loader
 
     def num_sections(self):
         """ Number of sections in the file
@@ -172,6 +182,13 @@ class ELFFile(object):
         if self._section_name_map is None:
             self._make_section_name_map()
         return self._section_name_map.get(section_name, None)
+    
+    def has_section(self, section_name):
+        """ Section existence check by name, without the overhead of parsing if found.
+        """
+        if self._section_name_map is None:
+            self._make_section_name_map()
+        return section_name in self._section_name_map
 
     def iter_sections(self, type=None):
         """ Yield all the sections in the file. If the optional |type|
@@ -231,14 +248,18 @@ class ELFFile(object):
                 end <= seg['p_vaddr'] + seg['p_filesz']):
                 yield start - seg['p_vaddr'] + seg['p_offset']
 
-    def has_dwarf_info(self):
+    def has_dwarf_info(self, strict=False):
         """ Check whether this file appears to have debugging information.
             We assume that if it has the .debug_info or .zdebug_info section, it
             has all the other required sections as well.
+
+            Unless you pass strict=True, the presence of .eh_frame section,
+            which is DWARF adjacent but hardly DWARF proper, will count as debug info.
+            Stripped files contain .eh_frame but none of the .[z]debug_xxx sections.
         """
-        return bool(self.get_section_by_name('.debug_info') or
-            self.get_section_by_name('.zdebug_info') or
-            self.get_section_by_name('.eh_frame'))
+        return (self.has_section('.debug_info') or
+            self.has_section('.zdebug_info') or
+            (not strict and self.has_section('.eh_frame')))
 
     def get_dwarf_info(self, relocate_dwarf_sections=True, follow_links=True):
         """ Return a DWARFInfo object representing the debugging information in
@@ -247,23 +268,39 @@ class ELFFile(object):
             If relocate_dwarf_sections is True, relocations for DWARF sections
             are looked up and applied.
 
-            If follow_links is True, we will try to load the supplementary
+            If follow_links is True, we will try to load the external and/or supplementary
             object file (if any), and use it to resolve references and imports.
         """
-        # Expect that has_dwarf_info was called, so at least .debug_info is
+        # Expect that has_dwarf_info() was called, so at least .debug_info is
         # present.
         # Sections that aren't found will be passed as None to DWARFInfo.
 
+        # TODO: support linking by build ID
+        # https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html
+
+        # A file may contain a debug link but not be stripped, so check for debug_info just in case
+        debuglink_section = self.get_section_by_name('.gnu_debuglink')
+        if debuglink_section and not self.has_dwarf_info(True) and follow_links and self.stream_loader:
+            debuglink = struct_parse(self.structs.Gnu_debuglink, debuglink_section.stream, debuglink_section.header.sh_offset)
+            with self.stream_loader(debuglink.filename) as ext_file:
+                # Validate checksum...
+                if _file_crc32(ext_file) != debuglink.checksum:
+                    raise ELFError('The linked DWARF file does not match the checksum in the link.')
+                ext_file.seek(0, os.SEEK_SET)
+                ext_elffile = ELFFile(ext_file, self.stream_loader)
+                # Inheriting the stream loader like that might be wrong if the supplementary DWARF link in the other file
+                # is relative to the other file's directory as opposed to this file's directory.
+                return ext_elffile.get_dwarf_info(relocate_dwarf_sections=relocate_dwarf_sections, follow_links=True)
+       
         section_names = ('.debug_info', '.debug_aranges', '.debug_abbrev',
                          '.debug_str', '.debug_line', '.debug_frame',
                          '.debug_loc', '.debug_ranges', '.debug_pubtypes',
                          '.debug_pubnames', '.debug_addr',
                          '.debug_str_offsets', '.debug_line_str',
                          '.debug_loclists', '.debug_rnglists',
-                         '.debug_sup', '.gnu_debugaltlink', '.gnu_debuglink',
-                         '.debug_types')
+                         '.debug_sup', '.gnu_debugaltlink', '.debug_types')
 
-        compressed = bool(self.get_section_by_name('.zdebug_info'))
+        compressed = self.has_section('.zdebug_info')
         if compressed:
             section_names = tuple(map(lambda x: '.z' + x[1:], section_names))
 
@@ -275,7 +312,7 @@ class ELFFile(object):
          debug_loc_sec_name, debug_ranges_sec_name, debug_pubtypes_name,
          debug_pubnames_name, debug_addr_name, debug_str_offsets_name,
          debug_line_str_name, debug_loclists_sec_name, debug_rnglists_sec_name,
-         debug_sup_name, gnu_debugaltlink_name, gnu_debuglink, debug_types_sec_name,
+         debug_sup_name, gnu_debugaltlink_name, debug_types_sec_name,
          eh_frame_sec_name) = section_names
 
         debug_sections = {}
@@ -318,13 +355,23 @@ class ELFFile(object):
                 debug_rnglists_sec=debug_sections[debug_rnglists_sec_name],
                 debug_sup_sec=debug_sections[debug_sup_name],
                 gnu_debugaltlink_sec=debug_sections[gnu_debugaltlink_name],
-                gnu_debuglink_sec=debug_sections[gnu_debuglink],
                 debug_types_sec=debug_sections[debug_types_sec_name]
                 )
         if follow_links:
             dwarfinfo.supplementary_dwarfinfo = self.get_supplementary_dwarfinfo(dwarfinfo)
         return dwarfinfo
-
+    
+    def has_dwarf_link(self):
+        """ Whether the binary's debug info is in an
+            external file. Use get_dwarf_link to retrieve the path to it.
+        """
+        return self.has_section('.gnu_debuglink')
+    
+    def get_dwarf_link(self):
+        """ Read the .gnu_debuglink section, return an object with filename (as bytes) and checksum (as number) in it.
+        """
+        section = self.get_section_by_name('.gnu_debuglink')
+        return struct_parse(self.structs.Gnu_debuglink, section.stream, section.header.sh_offset) if section else None
 
     def get_supplementary_dwarfinfo(self, dwarfinfo):
         """
diff --git a/elftools/elf/structs.py b/elftools/elf/structs.py
index 2af74aa..3a7ca66 100644
--- a/elftools/elf/structs.py
+++ b/elftools/elf/structs.py
@@ -112,6 +112,7 @@ class ELFStructs(object):
         self._create_riscv_attributes()
         self._create_elf_hash()
         self._create_gnu_hash()
+        self._create_gnu_debuglink()
 
     #-------------------------------- PRIVATE --------------------------------#
 
@@ -562,3 +563,9 @@ class ELFStructs(object):
                                self.Elf_word('bloom_shift'),
                                Array(lambda ctx: ctx['bloom_size'], self.Elf_xword('bloom')),
                                Array(lambda ctx: ctx['nbuckets'], self.Elf_word('buckets')))
+        
+    def _create_gnu_debuglink(self):
+        self.Gnu_debuglink = Struct('Gnu_debuglink',
+            CString("filename"),
+            Padding(lambda ctx: 3 - len(ctx.filename) % 4, strict=True),
+            self.Elf_word("checksum"))


Test patch:
diff --git a/test/test_debuglink.py b/test/test_debuglink.py
index b12baa6..932ea25 100644
--- a/test/test_debuglink.py
+++ b/test/test_debuglink.py
@@ -44,8 +44,8 @@ class TestDebuglink(unittest.TestCase):
         # Retrieve the subprograms from the DWARF info
         dwarf_info = elf.get_dwarf_info(follow_links=True, relocate_dwarf_sections=True)
 
-        if dwarf_info.supplementary_dwarfinfo:
-            for CU in dwarf_info.supplementary_dwarfinfo.iter_CUs():
+        if dwarf_info:
+            for CU in dwarf_info.iter_CUs():
                 for DIE in CU.iter_DIEs():
                     if DIE.tag == 'DW_TAG_subprogram':
                         attributes = DIE.attributes
@@ -70,9 +70,24 @@ class TestDebuglink(unittest.TestCase):
     def test_debuglink(self):
         with open('test/testfiles_for_unittests/debuglink', "rb") as elf_file:
             elf = ELFFile(elf_file, stream_loader=self.stream_loader)
+            # Contains eh_frame and gnu_debuglink, but no DWARF
+            self.assertTrue(elf.has_dwarf_info(False))
+            self.assertFalse(elf.has_dwarf_info(True))
+            self.assertTrue(elf.has_dwarf_link())
+
+            link = elf.get_dwarf_link()
+            self.assertIsNotNone(link)
+            self.assertEqual(link.filename, b'debuglink.debug')
+            self.assertEqual(link.checksum, 0x29b7c5f1)
+
             subprograms = self.subprograms_from_debuglink(elf)
             self.assertEqual(subprograms, {b'main': (0x1161, 0x52), b'addNumbers': (0x1149, 0x18)})
 
+        # Test the filesystem aware ELFFile loading
+        elf = ELFFile.load_from_path('test/testfiles_for_unittests/debuglink')
+        subprograms = self.subprograms_from_debuglink(elf)
+        self.assertEqual(subprograms, {b'main': (0x1161, 0x52), b'addNumbers': (0x1149, 0x18)})
+
 if __name__ == '__main__':
     unittest.main()
 
diff --git a/test/test_refaddr_bitness.py b/test/test_refaddr_bitness.py
index df05685..26ac508 100644
--- a/test/test_refaddr_bitness.py
+++ b/test/test_refaddr_bitness.py
@@ -51,7 +51,6 @@ class TestRefAddrOnDWARFv2With64BitTarget(unittest.TestCase):
             debug_rnglists_sec = None,
             debug_sup_sec = None,
             gnu_debugaltlink_sec = None,
-            gnu_debuglink_sec = None,
             debug_types_sec=None
         )