Repository: microsoft/ptvsd

Issue:
VSCode hangs when debugging with step into (F11) import statement
Original issue can be found here https://github.com/Microsoft/vscode-python/issues/3069


Discussion:
Looks like we don't receive `CMD_THREAD_SUSPEND_SINGLE_NOTIFICATION`, see last `->`:
`<-` == Request sent to pydevd
`->` == Received from pydevd

```console
<- 501 (CMD_VERSION)            [1000000000]: |1.1      WINDOWS ID|
-> 501 (CMD_VERSION)            [1000000000]: |@@BUILD_NUMBER@@\n|
<- 111 (CMD_SET_BREAK)          [1000000001]: |1        python-line     C:\GIT\pyscratch2\test&mytest\mytest.py 1       None    None    None    None    None    ALL|
<- 501 (CMD_VERSION)            [1000000002]: |1.1      WINDOWS ID|
<- 111 (CMD_SET_BREAK)          [1000000003]: |2        python-line     C:\GIT\pyscratch2\scratch1.py   1       None    None    None    None    None    ALL|
<- 501 (CMD_VERSION)            [1000000004]: |1.1      WINDOWS ID|
-> 501 (CMD_VERSION)            [1000000002]: |@@BUILD_NUMBER@@\n|
-> 501 (CMD_VERSION)            [1000000004]: |@@BUILD_NUMBER@@\n|
<- 122 (CMD_ADD_EXCEPTION_BREAK) [1000000005]: |python-BaseException    0       1       1|
<- 200 (CMD_REDIRECT_OUTPUT)    [1000000006]: |STDOUT   STDERR|
<- 154 (CMD_STOP_ON_START)      [1000000007]: |1|
<- 156 (CMD_PYDEVD_JSON_CONFIG) [1000000008]: |{"skip_print_breakpoint_exception": ["NameError"], "multi_threads_single_notification": true, "skip_suspend_on_breakpoint_exception": ["BaseException"]}|
<- 101 (CMD_RUN)                [1000000009]: ||
ready to debug
<- 102 (CMD_LIST_THREADS)       [1000000010]: ||
-> 502 (CMD_RETURN)             [1000000010]: |<xml><thread name="MainThread" id="pid_7716_id_42200208" /></xml>\n|
-> 103 (CMD_THREAD_CREATE)      [         2]: |<xml><thread name="MainThread" id="pid_7716_id_42200208" /></xml>\n|
-> 105 (CMD_THREAD_SUSPEND)     [         4]: |<xml><thread id="pid_7716_id_42200208" stop_reason="144" message="" suspend_type="trace"><frame id="54285712" name="&lt;module&gt;" file="C%3A%5CGIT%5Cpyscratch2%5Cmytest.py" line="1"></frame><frame id="53189296" name="_run" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="140"></frame><frame id="53148784" name="run_file" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="79"></frame><frame id="54073600" name="debug_main" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="45"></frame><frame id="53188528" name="handle_args" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C__main__.py" line="258"></frame><frame id="53148000" name="main" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C__main__.py" line="265"></frame><frame id="41801840" name="&lt;module&gt;" file="C%3A%5CUsers%5Ckanadig%5C.vscode%5Cextensions%5Cms-python.python-2018.10.1%5CpythonFiles%5Cexperimental%5Cptvsd_launcher.py" line="38"></frame></thread></xml>\n|
-> 157 (CMD_THREAD_SUSPEND_SINGLE_NOTIFICATION) [         6]: |{"stop_reason": 144, "thread_id": "pid_7716_id_42200208"}\n|
<- 102 (CMD_LIST_THREADS)       [1000000011]: ||
-> 502 (CMD_RETURN)             [1000000011]: |<xml><thread name="MainThread" id="pid_7716_id_42200208" /></xml>\n|
<- 152 (CMD_GET_THREAD_STACK)   [1000000012]: |pid_7716_id_42200208|
-> 152 (CMD_GET_THREAD_STACK)   [1000000012]: |<xml><thread id="pid_7716_id_42200208"><frame id="54285712" name="&lt;module&gt;" file="C%3A%5CGIT%5Cpyscratch2%5Cmytest.py" line="1"></frame><frame id="53189296" name="_run" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="140"></frame><frame id="53148784" name="run_file" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="79"></frame><frame id="54073600" name="debug_main" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="45"></frame><frame id="53188528" name="handle_args" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C__main__.py" line="258"></frame><frame id="53148000" name="main" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C__main__.py" line="265"></frame><frame id="41801840" name="&lt;module&gt;" file="C%3A%5CUsers%5Ckanadig%5C.vscode%5Cextensions%5Cms-python.python-2018.10.1%5CpythonFiles%5Cexperimental%5Cptvsd_launcher.py" line="38"></frame></thread></xml>\n|
<- 102 (CMD_LIST_THREADS)       [1000000013]: ||
-> 502 (CMD_RETURN)             [1000000013]: |<xml><thread name="MainThread" id="pid_7716_id_42200208" /></xml>\n|
<- 114 (CMD_GET_FRAME)          [1000000014]: |pid_7716_id_42200208     54285712        FRAME|
-> 114 (CMD_GET_FRAME)          [1000000014]: |<xml><var name="__builtins__" type="dict" qualifier="__builtin__" value="%7B%27ArithmeticError%27%3A %3Ctype %27exceptions.Ar...ticError%27&gt;%2C %27AssertionError%27%3A %3Ctype %27exceptions.As...ionError%27&gt;%2C %27AttributeError%27%3A %3Ctype %27exceptions.At...uteError%27&gt;%2C %27BaseException%27%3A %3Ctype %27exceptions.Ba...xception%27&gt;%2C %27BufferError%27%3A %3Ctype %27exceptions.Bu...ferError%27&gt;%2C %27BytesWarning%27%3A %3Ctype %27exceptions.By...sWarning%27&gt;%2C %27DeprecationWarning%27%3A %3Ctype %27exceptions.De...nWarning%27&gt;%2C %27EOFError%27%3A %3Ctype %27exceptions.EOFError%27&gt;%2C %27Ellipsis%27%3A Ellipsis%2C %27EnvironmentError%27%3A %3Ctype %27exceptions.En...entError%27&gt;%2C %27Exception%27%3A %3Ctype %27exceptions.Exception%27&gt;%2C %27False%27%3A False%2C %27FloatingPointError%27%3A %3Ctype %27exceptions.Fl...intError%27&gt;%2C %27FutureWarning%27%3A %3Ctype %27exceptions.Fu...eWarning%27&gt;%2C ...%7D" isContainer="True" />\n<var name="__doc__" type="NoneType" qualifier="__builtin__" value="None" />\n<var name="__file__" type="str" qualifier="__builtin__" value="%27c%3A%5C%5CGIT%5C%5Cpyscratch2%5C%5Cmytest.py%27" />\n<var name="__name__" type="str" qualifier="__builtin__" value="%27__main__%27" />\n<var name="__package__" type="NoneType" qualifier="__builtin__" value="None" />\n</xml>\n|
<- 144 (CMD_STEP_INTO_MY_CODE)  [1000000015]: |pid_7716_id_42200208|
-> 106 (CMD_THREAD_RUN)         [         8]: |pid_7716_id_42200208     144\n|
-> 158 (CMD_THREAD_RESUME_SINGLE_NOTIFICATION) [        10]: |{"thread_id": "pid_7716_id_42200208"}\n|
-> 105 (CMD_THREAD_SUSPEND)     [        12]: |<xml><thread id="pid_7716_id_42200208" stop_reason="111" message="" suspend_type="trace"><frame id="54465688" name="&lt;module&gt;" file="C%3A%5CGIT%5Cpyscratch2%5Cscratch1.py" line="1"></frame><frame id="54285712" name="&lt;module&gt;" file="C%3A%5CGIT%5Cpyscratch2%5Cmytest.py" line="1"></frame><frame id="53189296" name="_run" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="140"></frame><frame id="53148784" name="run_file" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="79"></frame><frame id="54073600" name="debug_main" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C_local.py" line="45"></frame><frame id="53188528" name="handle_args" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C__main__.py" line="258"></frame><frame id="53148000" name="main" file="C%3A%5CGIT%5Cptvsd%5Csrc%5Cptvsd%5C__main__.py" line="265"></frame><frame id="41801840" name="&lt;module&gt;" file="C%3A%5CUsers%5Ckanadig%5C.vscode%5Cextensions%5Cms-python.python-2018.10.1%5CpythonFiles%5Cexperimental%5Cptvsd_launcher.py" line="38"></frame></thread></xml>\n|
```
That's a bit odd... I'm able to reproduce the issue, but printing commands from pydevd I see the `CMD_THREAD_SUSPEND_SINGLE_NOTIFICATION` being written. 

Anyways, just wanted to note that I'm still investigating.
Ok, I've finished investigating but haven't done a fix yet because the issue lies on `ptvsd` and not on `pydevd` and I'm not sure of the best fix for it (fixes could be separating `ptvsd` and `pydevd` into 2 processes or making sure that `ptvsd` does not do any import while unmarshalling xml messages).

Now, on to the problem... the problem is that to handle a xml message, `ptvsd` uses `untangle` and `untangle` will try to do an `import` (when creating the xml sax parser) and the `import` will actually deadlock that thread until the tracing of the thread which is inside an import resumes.

It goes something like:

`user thread`: 
  -> import statement being evaluated
  -> pydevd detects step in, pauses and sends a command notifying that it was suspended (to be handled in the writer thread).
 
`writer thread`:
  -> receives suspend command
  -> uses untangle to unmarshal data
  -> untangle tries to create xml parser which does an import
  -> import does not complete because the user thread is stopped in a tracing function inside an import (any import in any thread would halt in this situation).

This is actually a CPython 2.7 gotcha (although it can probably be considered a bug too, but it'll surely never be fixed there) and is the reason `pydevd` itself imports everything it may need in the module top-level and not lazily inside of functions.

I've created a project on Github: https://github.com/fabioz/python27_locks_all_threads_on_import showing the problem without the debugger, just setting a tracing function and in that same repo another example showing the issue on the debugger where it becomes deadlocked.

The actual threads dump with the deadlock can be seen at the link below (see `Thread MainThread` and `Thread pydevd.Writer`):

https://raw.githubusercontent.com/fabioz/python27_locks_all_threads_on_import/master/example_showing_halted_thread_on_debugger/threads.log


Another thing that occurred to me is that a possible solution could be switching the pydevd to use the json messages (which is something already planned although not currently high priority) as I believe json unmarshaling doesn't do any imports.
The import that deadlocks seems to be in `xml.sax.xmlreader.IncrementalParser`, see definition for parse:
```py
    def parse(self, source):
        import saxutils  # <--- this import is the culprit.
        source = saxutils.prepare_input_source(source)
```
Note: `make_parse` in `xml.sax.__init__.py` also has a call  to `__import__` that we can avoid be doing this in `untangle.py`. 
```py
def make_parser():
    return  xml.sax.expatreader.create_parser()
```

so yep, using json might be one solution here.
I'll start to take a look into converting the messages to json.
@karthiknadig this one isn't really fixed (we have to port all messages because any message which is using `untangle` may hit the issue, so, this is still open, the pull request was only the initial proof of concept for porting the messages).
any update?
This is being worked on (although it may still take a while as it's a pretty big task).
any workaround meanwhile? debugging with print statements feels a bit archaic... :)
Well, the real workaround would be upgrading to Python 3 (as far as I now, this is a CPython 2 only bug which deadlocks if an import is done in a different thread and a breakpoint is added in the middle of an import -- see: https://github.com/Microsoft/ptvsd/issues/1039#issuecomment-442816437 for the explanation).
@fabioz , oh ok, we are soon in the process of upgrading to Python 3, so I consider this as good news! Thanks for the pointer!

Resolving patch:
diff --git a/src/ptvsd/untangle.py b/src/ptvsd/untangle.py
deleted file mode 100644
--- a/src/ptvsd/untangle.py
+++ /dev/null
@@ -1,214 +0,0 @@
-#!/usr/bin/env python
-
-"""
- untangle
-
- Converts xml to python objects.
-
- The only method you need to call is parse()
-
- Partially inspired by xml2obj
- (http://code.activestate.com/recipes/149368-xml2obj/)
-
- Author: Christian Stefanescu (http://0chris.com)
- License: MIT License - http://www.opensource.org/licenses/mit-license.php
-"""
-import os
-import keyword
-from xml.sax import make_parser, handler
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
-try:
-    from types import StringTypes
-
-    def is_string(x):
-        return isinstance(x, StringTypes)
-except ImportError:
-    def is_string(x):
-        return isinstance(x, str)
-
-__version__ = '1.1.1'
-
-
-class Element(object):
-    """
-    Representation of an XML element.
-    """
-    def __init__(self, name, attributes):
-        self._name = name
-        self._attributes = attributes
-        self.children = []
-        self.is_root = False
-        self.cdata = ''
-
-    def add_child(self, element):
-        """
-        Store child elements.
-        """
-        self.children.append(element)
-
-    def add_cdata(self, cdata):
-        """
-        Store cdata
-        """
-        self.cdata = self.cdata + cdata
-
-    def get_attribute(self, key):
-        """
-        Get attributes by key
-        """
-        return self._attributes.get(key)
-
-    def get_elements(self, name=None):
-        """
-        Find a child element by name
-        """
-        if name:
-            return [e for e in self.children if e._name == name]
-        else:
-            return self.children
-
-    def __getitem__(self, key):
-        return self.get_attribute(key)
-
-    def __getattr__(self, key):
-        matching_children = [x for x in self.children if x._name == key]
-        if matching_children:
-            if len(matching_children) == 1:
-                self.__dict__[key] = matching_children[0]
-                return matching_children[0]
-            else:
-                self.__dict__[key] = matching_children
-                return matching_children
-        else:
-            raise AttributeError(
-                "'%s' has no attribute '%s'" % (self._name, key)
-            )
-
-    def __hasattribute__(self, name):
-        if name in self.__dict__:
-            return True
-        return any(self.children, lambda x: x._name == name)
-
-    def __iter__(self):
-        yield self
-
-    def __str__(self):
-        return (
-            "Element <%s> with attributes %s, children %s and cdata %s" %
-            (self._name, self._attributes, self.children, self.cdata)
-        )
-
-    def __repr__(self):
-        return (
-            "Element(name = %s, attributes = %s, cdata = %s)" %
-            (self._name, self._attributes, self.cdata)
-        )
-
-    def __nonzero__(self):
-        return self.is_root or self._name is not None
-
-    def __eq__(self, val):
-        return self.cdata == val
-
-    def __dir__(self):
-        children_names = [x._name for x in self.children]
-        return children_names
-
-    def __len__(self):
-        return len(self.children)
-
-    def __contains__(self, key):
-        return key in dir(self)
-
-
-class Handler(handler.ContentHandler):
-    """
-    SAX handler which creates the Python object structure out of ``Element``s
-    """
-    def __init__(self):
-        self.root = Element(None, None)
-        self.root.is_root = True
-        self.elements = []
-
-    def startElement(self, name, attributes):
-        name = name.replace('-', '_')
-        name = name.replace('.', '_')
-        name = name.replace(':', '_')
-
-        # adding trailing _ for keywords
-        if keyword.iskeyword(name):
-            name += '_'
-
-        attrs = dict()
-        for k, v in attributes.items():
-            attrs[k] = v
-        element = Element(name, attrs)
-        if len(self.elements) > 0:
-            self.elements[-1].add_child(element)
-        else:
-            self.root.add_child(element)
-        self.elements.append(element)
-
-    def endElement(self, name):
-        self.elements.pop()
-
-    def characters(self, cdata):
-        self.elements[-1].add_cdata(cdata)
-
-
-def parse(filename, **parser_features):
-    """
-    Interprets the given string as a filename, URL or XML data string,
-    parses it and returns a Python object which represents the given
-    document.
-
-    Extra arguments to this function are treated as feature values to pass
-    to ``parser.setFeature()``. For example, ``feature_external_ges=False``
-    will set ``xml.sax.handler.feature_external_ges`` to False, disabling
-    the parser's inclusion of external general (text) entities such as DTDs.
-
-    Raises ``ValueError`` if the first argument is None / empty string.
-
-    Raises ``AttributeError`` if a requested xml.sax feature is not found in
-    ``xml.sax.handler``.
-
-    Raises ``xml.sax.SAXParseException`` if something goes wrong
-    during parsing.
-    """
-    if (filename is None or (is_string(filename) and filename.strip()) == ''):
-        raise ValueError('parse() takes a filename, URL or XML string')
-    parser = make_parser()
-    for feature, value in parser_features.items():
-        parser.setFeature(getattr(handler, feature), value)
-    sax_handler = Handler()
-    parser.setContentHandler(sax_handler)
-    is_filename = False
-    try:
-        if is_string(filename):
-            is_filename = os.path.exists(filename) or is_url(filename)
-    except ValueError:
-        pass
-    if is_filename:
-        parser.parse(filename)
-    else:
-        if hasattr(filename, 'read'):
-            parser.parse(filename)
-        else:
-            parser.parse(StringIO(filename))
-
-    return sax_handler.root
-
-
-def is_url(string):
-    """
-    Checks if the given string starts with 'http(s)'.
-    """
-    try:
-        return string.startswith('http://') or string.startswith('https://')
-    except AttributeError:
-        return False
-
-# vim: set expandtab ts=4 sw=4:
diff --git a/src/ptvsd/wrapper.py b/src/ptvsd/wrapper.py
--- a/src/ptvsd/wrapper.py
+++ b/src/ptvsd/wrapper.py
@@ -7,7 +7,6 @@
 import bisect
 import copy
 import errno
-import io
 import json
 import os
 import platform
@@ -22,13 +21,13 @@
     urllib.unquote
 except Exception:
     import urllib.parse as urllib
+
 try:
     import queue
 except ImportError:
     import Queue as queue
-import warnings
-from xml.sax.saxutils import unescape as xml_unescape
 
+import warnings
 import pydevd  # noqa
 import _pydevd_bundle.pydevd_comm as pydevd_comm  # noqa
 import _pydevd_bundle.pydevd_comm_constants as pydevd_comm_constants  # noqa
@@ -47,7 +46,6 @@
 from ptvsd.compat import unicode
 import ptvsd.ipcjson as ipcjson  # noqa
 import ptvsd.futures as futures  # noqa
-import ptvsd.untangle as untangle  # noqa
 from ptvsd.pathutils import PathUnNormcase  # noqa
 from ptvsd.version import __version__  # noqa
 from ptvsd.socket import TimeoutError  # noqa
@@ -158,15 +156,6 @@ def unquote(s):
         return s
 
 
-def unquote_xml_path(s):
-    """XML unescape after url unquote. This reverses the escapes and quotes done
-    by pydevd.
-    """
-    if s is None:
-        return None
-    return xml_unescape(unquote(s))
-
-
 class IDMap(object):
     """Maps VSCode entities to corresponding pydevd entities by ID.
 
@@ -1352,10 +1341,6 @@ def on_pydevd_event(self, cmd_id, seq, args):
         else:
             return None
 
-    @staticmethod
-    def parse_xml_response(args):
-        return untangle.parse(io.BytesIO(args.encode('utf8'))).xml
-
     def _wait_for_pydevd_ready(self):
         # TODO: Call self._ensure_pydevd_requests_handled?
         pass


Test patch:
diff --git a/tests/test_internals_filter.py b/tests/test_internals_filter.py
--- a/tests/test_internals_filter.py
+++ b/tests/test_internals_filter.py
@@ -4,14 +4,14 @@
 
 import os
 import pytest
-import ptvsd.untangle
+import ptvsd
 
 from ptvsd.wrapper import InternalsFilter
 from ptvsd.wrapper import dont_trace_ptvsd_files
 
-INTERNAL_DIR = os.path.dirname(os.path.abspath(ptvsd.untangle.__file__))
+INTERNAL_DIR = os.path.dirname(os.path.abspath(ptvsd.__file__))
 @pytest.mark.parametrize('path', [
-    os.path.abspath(ptvsd.untangle.__file__),
+    os.path.abspath(ptvsd.__file__),
     # File used by VS/VSC to launch ptvsd
     os.path.join('somepath', 'ptvsd_launcher.py'),
     # Any file under ptvsd
@@ -42,4 +42,4 @@ def test_user_file_paths(path):
     (os.path.join('usr', 'abcd', 'ptvsd', 'tangle.py'), False),
 ])
 def test_ptvsd_paths(path, val):
-    assert val == dont_trace_ptvsd_files(path)
+    assert val == dont_trace_ptvsd_files(os.path.normcase(path))