Repository: microsoft/ptvsd

Issue:
Add support for remote source references
Adds support for [sourceReference](https://github.com/Microsoft/vscode-debugadapter-node/blob/master/protocol/src/debugProtocol.ts#L818)

**Scenarios:**
1. Debugging std libs or similar
2. Debug a remote application without any source on the client side (all we need is, is one of, `breakpoint()`, `ptvsd.wait_for_attach` or `ptvsd.break_into_debugger ` or `an exception`).

I'd consider item 2 to be a great feature for release (I'm assuming this isn't already possible in PTVS, I know it isn't in VSC Python).

**When does this come into play:**
Only when remote debugging (`attach`) and if ptvsd is unable to map the server file to a client file (i.e. PTVSD is unable to map the file)

**Status**
* I've got this working, just need to test on Windows, Linux and a docker container
* Seems to work without any source
* Breakpoints/pause/breaking into exceptions (works)

**Items to consider/check:**
* UX (latency) when debugging large files


@karthiknadig @int19h 
Any thoughts/suggestions?


Discussion:
This wasn't a supported scenario for the old debugger in VS either, but it was always on the wish list. It certainly simplifies things quite a bit in exception debugging scenarios (which is pretty common for remote), since you don't need any local sources at all to attach and break on exception. We still need path mapping for breakpoints set in advance, so this can't fully replace that, but it's very useful even so.

Resolving patch:
diff --git a/ptvsd/wrapper.py b/ptvsd/wrapper.py
--- a/ptvsd/wrapper.py
+++ b/ptvsd/wrapper.py
@@ -625,6 +625,8 @@ def __init__(self, socket, pydevd_notify, pydevd_request,
         self.frame_map = IDMap()
         self.var_map = IDMap()
         self.bp_map = IDMap()
+        self.source_map = IDMap()
+        self.enable_source_references = False
         self.next_var_ref = 0
         self.exceptions_mgr = ExceptionsManager(self)
         self.modules_mgr = ModulesManager(self)
@@ -890,7 +892,7 @@ def build_debug_options(self, debug_options):
             'Flask': 'FLASK_DEBUG=True',
             'Jinja': 'FLASK_DEBUG=True',
             'FixFilePathCase': 'FIX_FILE_PATH_CASE=True',
-            'DebugStdLib': 'DEBUG_STD_LIB=True'
+            'DebugStdLib': 'DEBUG_STD_LIB=True',
         }
         return ';'.join(debug_option_mapping[option]
                         for option in debug_options
@@ -915,7 +917,7 @@ def _parse_debug_options(self, debug_options):
             'WEB_BROWSER_URL': unquote,
             'DJANGO_DEBUG': bool,
             'FLASK_DEBUG': bool,
-            'FIX_FILE_PATH_CASE': bool
+            'FIX_FILE_PATH_CASE': bool,
         }
 
         options = {}
@@ -1027,6 +1029,53 @@ def on_threads(self, request, args):
 
         self.send_response(request, threads=threads)
 
+    @async_handler
+    def on_source(self, request, args):
+        """Request to get the source"""
+        source_reference = args.get('sourceReference', 0)
+        filename = '' if source_reference == 0 else \
+            self.source_map.to_pydevd(source_reference)
+
+        if source_reference == 0:
+            self.send_error_response(request, 'Source unavailable')
+        else:
+            server_filename = pydevd_file_utils.norm_file_to_server(filename)
+
+            cmd = pydevd_comm.CMD_LOAD_SOURCE
+            _, _, content = yield self.pydevd_request(cmd, server_filename)
+            self.send_response(request, content=content)
+
+    def get_source_reference(self, filename):
+        """Gets the source reference only in remote debugging scenarios.
+        And we know that the path returned is the same as the server path
+        (i.e. path has not been translated)"""
+
+        if self.start_reason == 'launch':
+            return 0
+
+        try:
+            return self.source_map.to_vscode(filename, autogen=False)
+        except KeyError:
+            pass
+
+        server_filename = pydevd_file_utils.norm_file_to_server(filename)
+
+        # If the mapped file is the same as the file we provided,
+        # then we can generate a soure reference.
+        if server_filename == filename and os.path.exists(server_filename):
+            return self.source_map.to_vscode(filename, autogen=True)
+        elif platform.system() == 'Windows' and \
+            server_filename.upper() == filename.upper() and \
+            os.path.exists(server_filename):
+            return self.source_map.to_vscode(filename, autogen=True)
+        elif server_filename.replace('\\', '/') == filename.replace('\\', '/'):
+            # If remote is Unix and local is Windows, then PyDevD will
+            #   replace the path separator in remote with with
+            #   the os path separator of remote client
+            return self.source_map.to_vscode(filename, autogen=True)
+        else:
+            return 0
+
     @async_handler
     def on_stackTrace(self, request, args):
         # TODO: docstring
@@ -1062,6 +1111,7 @@ def on_stackTrace(self, request, args):
             fid = self.frame_map.to_vscode(key, autogen=True)
             name = unquote(xframe['name'])
             norm_path = self.path_casing.un_normcase(unquote(xframe['file']))
+            source_reference = self.get_source_reference(norm_path)
             module = self.modules_mgr.add_or_get_from_path(norm_path)
             line = int(xframe['line'])
             frame_name = self._format_frame_name(
@@ -1074,7 +1124,10 @@ def on_stackTrace(self, request, args):
             stackFrames.append({
                 'id': fid,
                 'name': frame_name,
-                'source': {'path': norm_path},
+                'source': {
+                    'path': norm_path,
+                    'sourceReference': source_reference
+                },
                 'line': line, 'column': 1,
             })
 


Test patch:
diff --git a/tests/highlevel/test_messages.py b/tests/highlevel/test_messages.py
--- a/tests/highlevel/test_messages.py
+++ b/tests/highlevel/test_messages.py
@@ -285,14 +285,14 @@ def test_basic(self):
                     {
                         'id': 1,
                         'name': 'spam',
-                        'source': {'path': 'abc.py'},
+                        'source': {'path': 'abc.py', 'sourceReference': 0},
                         'line': 10,
                         'column': 1,
                     },
                     {
                         'id': 2,
                         'name': 'eggs',
-                        'source': {'path': 'xyz.py'},
+                        'source': {'path': 'xyz.py', 'sourceReference': 0},
                         'line': 2,
                         'column': 1,
                     },
@@ -329,14 +329,14 @@ def test_with_frame_format(self):
                     {
                         'id': 1,
                         'name': 'abc.spam : 10',
-                        'source': {'path': 'abc.py'},
+                        'source': {'path': 'abc.py', 'sourceReference': 0},
                         'line': 10,
                         'column': 1,
                     },
                     {
                         'id': 2,
                         'name': 'xyz.eggs : 2',
-                        'source': {'path': 'xyz.py'},
+                        'source': {'path': 'xyz.py', 'sourceReference': 0},
                         'line': 2,
                         'column': 1,
                     },
@@ -1886,9 +1886,7 @@ def test_unsupported(self):
             )
             received = self.vsc.received
 
-        self.assert_vsc_received(received, [
-            self.expected_failure('Unknown command'),
-        ])
+        self.assert_vsc_received(received, [])
         self.assert_received(self.debugger, [])