Repository: getsentry/sentry

Issue:
Symbolication Failures on macOS
Looks like some users are reporting that symbolication is failing for them on macOS. The application frames just show `<unknown>` or `<redacted>`.

Tooltip says "with the tooltip: "Unknown system frame. Usually from beta SDKs". This might be a regression from the `in_app` detection. Unsure if server or client at this point thoug


Discussion:
The issue effectively is most likely that `is_image_from_app_bundle` in the symbolizer now decides hard on if we go through the symbolserver or the user dsyms whereas previously we always fell back to the user dsyms.  This regression was caused by adding support for reprocessing.

We cannot reintroduce the fallback but for non iOS builds we could trivially change the behavior of `is_image_from_app_bundle` to consider anything below a path named something like `\.app/Contents/` to be "in app".
Actually on second thought we might just consider everything app bundled on macOS since the symbolserver does not carry any mac symbols.

Resolving patch:
diff --git a/src/sentry/lang/native/plugin.py b/src/sentry/lang/native/plugin.py
--- a/src/sentry/lang/native/plugin.py
+++ b/src/sentry/lang/native/plugin.py
@@ -190,7 +190,7 @@ def process_frame(self, processable_frame, processing_task):
             # Construct a raw frame that is used by the symbolizer
             # backend.  We only assemble the bare minimum we need here.
             instruction_addr = processable_frame.data['instruction_addr']
-            in_app = self.sym.is_in_app(instruction_addr)
+            in_app = self.sym.is_in_app(instruction_addr, sdk_info=self.sdk_info)
             in_app = (in_app and not self.sym.is_internal_function(raw_frame.get('function')))
             if raw_frame.get('in_app') is None:
                 raw_frame['in_app'] = in_app
diff --git a/src/sentry/lang/native/symbolizer.py b/src/sentry/lang/native/symbolizer.py
--- a/src/sentry/lang/native/symbolizer.py
+++ b/src/sentry/lang/native/symbolizer.py
@@ -36,6 +36,7 @@
 ''')
 SIM_PATH = '/Developer/CoreSimulator/Devices/'
 SIM_APP_PATH = '/Containers/Bundle/Application/'
+MAC_OS_PATH = '.app/Contents/'
 
 _internal_function_re = re.compile(r'(kscm_|kscrash_|KSCrash |SentryClient |RNSentry )')
 
@@ -135,10 +136,13 @@ def _process_frame(self, frame, img):
 
         return frame
 
-    def is_image_from_app_bundle(self, img):
+    def is_image_from_app_bundle(self, img, sdk_info=None):
         fn = img['name']
+        is_mac_platform = (
+            sdk_info is not None and sdk_info['sdk_name'].lower() == 'macos')
         if not (fn.startswith(APP_BUNDLE_PATHS) or
-                (SIM_PATH in fn and SIM_APP_PATH in fn)):
+                (SIM_PATH in fn and SIM_APP_PATH in fn) or
+                (is_mac_platform and MAC_OS_PATH in fn)):
             return False
         return True
 
@@ -153,13 +157,13 @@ def _is_app_bundled_framework(self, img):
         fn = img['name']
         return fn.startswith(APP_BUNDLE_PATHS) and '/Frameworks/' in fn
 
-    def _is_app_frame(self, instruction_addr, img):
+    def _is_app_frame(self, instruction_addr, img, sdk_info=None):
         """Given a frame derives the value of `in_app` by discarding the
         original value of the frame.
         """
         # Anything that is outside the app bundle is definitely not a
         # frame from out app.
-        if not self.is_image_from_app_bundle(img):
+        if not self.is_image_from_app_bundle(img, sdk_info=sdk_info):
             return False
 
         # We also do not consider known support frameworks to be part of
@@ -170,11 +174,11 @@ def _is_app_frame(self, instruction_addr, img):
         # Otherwise, yeah, let's just say it's in_app
         return True
 
-    def _is_optional_dsym(self, img):
+    def _is_optional_dsym(self, img, sdk_info=None):
         """Checks if this is a dsym that is optional."""
         # Frames that are not in the app are not considered optional.  In
         # theory we should never reach this anyways.
-        if not self.is_image_from_app_bundle(img):
+        if not self.is_image_from_app_bundle(img, sdk_info=sdk_info):
             return False
 
         # If we're dealing with an app bundled framework that is also
@@ -193,10 +197,10 @@ def _is_optional_dsym(self, img):
     def _is_simulator_frame(self, frame, img):
         return _sim_platform_re.search(img['name']) is not None
 
-    def _symbolize_app_frame(self, instruction_addr, img):
+    def _symbolize_app_frame(self, instruction_addr, img, sdk_info=None):
         dsym_path = self.dsym_paths.get(img['uuid'])
         if dsym_path is None:
-            if self._is_optional_dsym(img):
+            if self._is_optional_dsym(img, sdk_info=sdk_info):
                 type = EventError.NATIVE_MISSING_OPTIONALLY_BUNDLED_DSYM
             else:
                 type = EventError.NATIVE_MISSING_DSYM
@@ -259,15 +263,17 @@ def symbolize_frame(self, instruction_addr, sdk_info=None,
         # If we are dealing with a frame that is not bundled with the app
         # we look at system symbols.  If that fails, we go to looking for
         # app symbols explicitly.
-        if not self.is_image_from_app_bundle(img):
+        if not self.is_image_from_app_bundle(img, sdk_info=sdk_info):
             return self._convert_symbolserver_match(instruction_addr,
                                                     symbolserver_match, img)
 
-        return self._symbolize_app_frame(instruction_addr, img)
+        return self._symbolize_app_frame(
+            instruction_addr, img, sdk_info=sdk_info)
 
-    def is_in_app(self, instruction_addr):
+    def is_in_app(self, instruction_addr, sdk_info=None):
         img = self.image_lookup.find_image(instruction_addr)
-        return img is not None and self._is_app_frame(instruction_addr, img)
+        return img is not None and self._is_app_frame(
+            instruction_addr, img, sdk_info=sdk_info)
 
     def is_internal_function(self, function):
         return _internal_function_re.search(function) is not None


Test patch:
diff --git a/tests/sentry/lang/native/test_plugin.py b/tests/sentry/lang/native/test_plugin.py
--- a/tests/sentry/lang/native/test_plugin.py
+++ b/tests/sentry/lang/native/test_plugin.py
@@ -185,7 +185,7 @@ def test_frame_resolution(self, symbolize_frame):
 
         assert len(event.interfaces['threads'].values) == 1
 
-    def sym_app_frame(self, instruction_addr, img):
+    def sym_app_frame(self, instruction_addr, img, sdk_info=None):
         object_name = (
             "/var/containers/Bundle/Application/"
             "B33C37A8-F933-4B6B-9FFA-152282BFDF13/"
@@ -543,7 +543,7 @@ def test_frame_resolution(self, symbolize_frame):
 
         assert len(event.interfaces['threads'].values) == 1
 
-    def sym_app_frame(self, instruction_addr, img):
+    def sym_app_frame(self, instruction_addr, img, sdk_info=None):
         object_name = (
             "/var/containers/Bundle/Application/"
             "B33C37A8-F933-4B6B-9FFA-152282BFDF13/"
@@ -881,3 +881,150 @@ def test_in_app_function_name(self):
         assert not frames[4].in_app
         assert frames[5].in_app
         assert frames[6].in_app
+
+    def sym_mac_app_frame(self, instruction_addr, img, sdk_info=None):
+        object_name = (
+            "/Users/haza/Library/Developer/Xcode/Archives/2017-06-19/"
+            "CrashProbe 19-06-2017, 08.53.xcarchive/Products/Applications/"
+            "CrashProbe.app/Contents/Frameworks/"
+            "CrashLib.framework/Versions/A/CrashLib"
+        )
+        if not (4295098384 <= parse_addr(instruction_addr) < 4295098388):
+            return [{
+                'filename': 'Foo.swift',
+                'abs_path': 'Foo.swift',
+                'lineno': 82,
+                'colno': 23,
+                'package': object_name,
+                'function': 'other_main',
+                'symbol_addr': '0x1',
+                "instruction_addr": '0x1',
+            }]
+        return [{
+            'filename': 'Foo.swift',
+            'abs_path': 'Foo.swift',
+            'lineno': 42,
+            'colno': 23,
+            'package': object_name,
+            'function': 'real_main',
+            'symbol_addr': '0x1000262a0',
+            "instruction_addr": '0x100026330',
+        }]
+
+    @patch.object(Symbolizer, '_symbolize_app_frame', sym_mac_app_frame)
+    def test_in_app_macos(self):
+        object_name = (
+            "/Users/haza/Library/Developer/Xcode/Archives/2017-06-19/"
+            "CrashProbe 19-06-2017, 08.53.xcarchive/Products/Applications/"
+            "CrashProbe.app/Contents/Frameworks/"
+            "CrashLib.framework/Versions/A/CrashLib"
+        )
+        event_data = {
+            "sentry.interfaces.User": {
+                "ip_address": "31.172.207.97"
+            },
+            "extra": {},
+            "project": self.project.id,
+            "platform": "cocoa",
+            "debug_meta": {
+                "images": [
+                    {
+                        "type": "apple",
+                        "cpu_subtype": 0,
+                        "uuid": "C05B4DDD-69A7-3840-A649-32180D341587",
+                        "image_vmaddr": 4294967296,
+                        "image_addr": 4295098368,
+                        "cpu_type": 16777228,
+                        "image_size": 32768,
+                        "name": object_name,
+                    }
+                ]
+            },
+            "sentry.interfaces.Exception": {
+                "values": [
+                    {
+                        "stacktrace": {
+                            "frames": [
+                                {
+                                    "function": "-[CRLCrashAsyncSafeThread crash]",
+                                    "abs_path": "/Users/haza/Projects/getsentry-CrashProbe/CrashProbe/CRLCrashAsyncSafeThread.m",
+                                    "package": "/Users/haza/Library/Developer/Xcode/Archives/2017-06-19/CrashProbe 19-06-2017, 08.53.xcarchive/Products/Applications/CrashProbe.app/Contents/Frameworks/CrashLib.framework/Versions/A/CrashLib",
+                                    "image_addr": "0x110121000",
+                                    "symbol_addr": "0x110122303",
+                                    "instruction_addr": 4295098388
+                                },
+                                {
+                                    "function": "[KSCrash ]",
+                                    "abs_path": None,
+                                    "package": "/usr/lib/system/libdyld.dylib",
+                                    "filename": None,
+                                    "symbol_addr": "0x002ac28b4",
+                                    "lineno": None,
+                                    "instruction_addr": 4295098388,
+                                },
+                            ]
+                        },
+                        "type": "NSRangeException",
+                        "mechanism": {
+                            "posix_signal": {
+                                "signal": 6,
+                                "code": 0,
+                                "name": "SIGABRT",
+                                "code_name": None
+                            },
+                            "type": "cocoa",
+                            "mach_exception": {
+                                "subcode": 0,
+                                "code": 0,
+                                "exception": 10,
+                                "exception_name": "EXC_CRASH"
+                            }
+                        },
+                        "value": (
+                            "*** -[__NSArray0 objectAtIndex:]: index 3 "
+                            "beyond bounds for empty NSArray"
+                        )
+                    }
+                ]
+            },
+            "contexts": {
+                "device": {
+                    "family": "macOS",
+                    "type": "device",
+                    "storage_size": 498954403840,
+                    "free_memory": 415174656,
+                    "memory_size": 17179869184,
+                    "boot_time": "2017-06-18T07:10:05Z",
+                    "model": "MacBookPro13,1",
+                    "usable_memory": 15204716544,
+                    "arch": "x86"
+                },
+                "app": {
+                    "app_version": "1.0",
+                    "app_name": "CrashProbe",
+                    "device_app_hash": "75e22adcce6cb4c81db7c7e623c2f2721616d2c8",
+                    "executable_path": "/Users/haza/Library/Developer/Xcode/Archives/2017-06-19/CrashProbe 19-06-2017, 08.53.xcarchive/Products/Applications/CrashProbe.app/CrashProbe",
+                    "build_type": "unknown",
+                    "app_start_time": "2017-06-19T07:19:02Z",
+                    "app_identifier": "net.hockeyapp.CrashProbe",
+                    "type": "app",
+                    "app_build": "1"
+                },
+                "os": {
+                    "rooted": False,
+                    "kernel_version": "Darwin Kernel Version 16.6.0: Fri Apr 14 16:21:16 PDT 2017; root:xnu-3789.60.24~6/RELEASE_X86_64",
+                    "version": "10.12.5",
+                    "build": "16F73",
+                    "type": "os",
+                    "name": "macOS"
+                }
+            },
+        }
+        resp = self._postWithHeader(event_data)
+        assert resp.status_code == 200
+
+        event = Event.objects.get()
+
+        bt = event.interfaces['sentry.interfaces.Exception'].values[0].stacktrace
+        frames = bt.frames
+        assert frames[0].in_app
diff --git a/tests/sentry/lang/native/test_processor.py b/tests/sentry/lang/native/test_processor.py
--- a/tests/sentry/lang/native/test_processor.py
+++ b/tests/sentry/lang/native/test_processor.py
@@ -31,7 +31,7 @@
 }
 
 
-def patched_symbolize_app_frame(self, instruction_addr, img):
+def patched_symbolize_app_frame(self, instruction_addr, img, sdk_info=None):
     return [{
         'filename': 'Foo.swift',
         'abs_path': 'Foo.swift',