Repository: microsoft/ptvsd

Issue:
enable_attach() hangs if called from top level of an imported module on Python 2.7
Reliable repro for me is any of the `test_run` attach tests, e.g. `test_run[AttachSocketImport-program]`.

The test tries to wait until it sees the server bringing the port up. Eventually it gives up and just tells the adapter to attach, which then fails with "No connection could be made because the target machine actively refused it".

Debuggee, meanwhile, is stuck in this stack (obtained via VS mixed-mode debugger):
```
 	threading.py!wait Line 340	Python
 	threading.py!wait Line 614	Python
 	pydevd.py!wait_for_server_socket_ready Line 1150	Python
>	pydevd.py!_enable_attach Line 2249	Python
 	api.py!enable_attach Line 72	Python
 	api.py!debug Line 34	Python
 	__init__.py!enable_attach Line 48	Python
 	<unknown>!<module> Line 3	Python
 	__init__.py!<module> Line 50	Python
 	code_to_debug.py!<module> Line 1	Python
```

The reason is that `enable_attach()` is called from the top-level of a module being imported - thus, the import lock is being held. And the background thread that this is waiting for is trying to do a bunch of imports, which needs the import lock. The result is a deadlock on 2.7 (3.x can handle this stuff fine).

`enable_attach()` from a module is a legitimate scenario - sometimes, when debugging Python code hosted in an app, there's no other convenient place to put it. If at all possible, we need to accommodate that option.

The following are the imports I found on this code path so far:

In `pydevd_comm.create_server_socket()`:
```py
from socket import SO_EXCLUSIVEADDRUSE
```

In `pydevd_comm.create_server_socket()`:
```py
server.bind((host, port))
```
(this one indirectly tries to load the "idna" codec to parse the hostname; we already preload some codecs in ptvsd for similar reasons, so I'll add it to that list)

In `pydevd_comm.ReaderThread.__init__`:
```py
from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
from _pydevd_bundle.pydevd_process_net_command import process_net_command
```

There may be more; I couldn't get past the last one, because it's not a trivial fix - moving those imports to top level causes a circular dependency between modules.


Discussion:
I'm taking a look into an actual fix here.

Resolving patch:
diff --git a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py
--- a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py
+++ b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py
@@ -115,6 +115,9 @@
 # CMD_XXX constants imported for backward compatibility
 from _pydevd_bundle.pydevd_comm_constants import *  # @UnusedWildImport
 
+if IS_WINDOWS:
+    from socket import SO_EXCLUSIVEADDRUSE
+
 if IS_JYTHON:
     import org.python.core as JyCore  # @UnresolvedImport
 
@@ -187,10 +190,8 @@ def run_as_pydevd_daemon_thread(func, *args, **kwargs):
 class ReaderThread(PyDBDaemonThread):
     ''' reader thread reads and dispatches commands in an infinite loop '''
 
-    def __init__(self, sock, py_db, terminate_on_socket_close=True):
+    def __init__(self, sock, py_db, PyDevJsonCommandProcessor, process_net_command, terminate_on_socket_close=True):
         assert sock is not None
-        from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
-        from _pydevd_bundle.pydevd_process_net_command import process_net_command
         PyDBDaemonThread.__init__(self)
         self._terminate_on_socket_close = terminate_on_socket_close
 
@@ -429,7 +430,6 @@ def create_server_socket(host, port):
     try:
         server = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)
         if IS_WINDOWS:
-            from socket import SO_EXCLUSIVEADDRUSE
             server.setsockopt(SOL_SOCKET, SO_EXCLUSIVEADDRUSE, 1)
         else:
             server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
diff --git a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
--- a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
+++ b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
@@ -18,6 +18,11 @@
 
 import sys  # Note: the sys import must be here anyways (others depend on it)
 
+# Preload codecs to avoid imports to them later on which can potentially halt the debugger.
+import codecs as _codecs
+for _codec in ["ascii", "utf8", "utf-8", "latin1", "latin-1", "idna"]:
+    _codecs.lookup(_codec)
+
 
 class DebugInfoHolder:
     # we have to put it here because it can be set through the command line (so, the
diff --git a/src/ptvsd/_vendored/pydevd/pydevd.py b/src/ptvsd/_vendored/pydevd/pydevd.py
--- a/src/ptvsd/_vendored/pydevd/pydevd.py
+++ b/src/ptvsd/_vendored/pydevd/pydevd.py
@@ -67,6 +67,9 @@
     start_client, start_server, InternalGetBreakpointException, InternalSendCurrExceptionTrace,
     InternalSendCurrExceptionTraceProceeded, run_as_pydevd_daemon_thread)
 
+from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
+from _pydevd_bundle.pydevd_process_net_command import process_net_command
+
 from _pydevd_bundle.pydevd_breakpoints import stop_on_unhandled_exception
 from _pydevd_bundle.pydevd_collect_try_except_info import collect_try_except_info
 from _pydevd_bundle.pydevd_suspended_frames import SuspendedFramesManager
@@ -1124,7 +1127,13 @@ def initialize_network(self, sock, terminate_on_socket_close=True):
             curr_writer.do_kill_pydev_thread()
 
         self.writer = WriterThread(sock, self, terminate_on_socket_close=terminate_on_socket_close)
-        self.reader = ReaderThread(sock, self, terminate_on_socket_close=terminate_on_socket_close)
+        self.reader = ReaderThread(
+            sock,
+            self,
+            PyDevJsonCommandProcessor=PyDevJsonCommandProcessor,
+            process_net_command=process_net_command,
+            terminate_on_socket_close=terminate_on_socket_close
+        )
         self.writer.start()
         self.reader.start()
 
@@ -2565,7 +2574,13 @@ class DispatchReader(ReaderThread):
 
     def __init__(self, dispatcher):
         self.dispatcher = dispatcher
-        ReaderThread.__init__(self, self.dispatcher.client)
+
+        ReaderThread.__init__(
+            self,
+            self.dispatcher.client,
+            PyDevJsonCommandProcessor=PyDevJsonCommandProcessor,
+            process_net_command=process_net_command,
+        )
 
     @overrides(ReaderThread._on_run)
     def _on_run(self):


Test patch:
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach.py b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach.py
@@ -1,50 +1,3 @@
 if __name__ == '__main__':
-    import os
-    import sys
-    import time
-    port = int(sys.argv[1])
-    root_dirname = os.path.dirname(os.path.dirname(__file__))
-
-    if root_dirname not in sys.path:
-        sys.path.append(root_dirname)
-
-    import pydevd
-    try:
-        pydevd._wait_for_attach()  # Cannot be called before _enable_attach.
-    except AssertionError:
-        pass
-    else:
-        raise AssertionError('Expected _wait_for_attach to raise exception.')
-
-    assert sys.gettrace() is None
-    print('enable attach to port: %s' % (port,))
-    pydevd._enable_attach(('127.0.0.1', port))
-    pydevd._enable_attach(('127.0.0.1', port))  # no-op in practice
-
-    try:
-        pydevd._enable_attach(('127.0.0.1', port + 15))  # different port: raise error.
-    except AssertionError:
-        pass
-    else:
-        raise AssertionError('Expected _enable_attach to raise exception (because it is already hearing in another port).')
-
-    assert pydevd.get_global_debugger() is not None
-    assert sys.gettrace() is not None
-
-    a = 10  # Break 1
-    print('wait for attach')
-    pydevd._wait_for_attach()
-    print('finished wait for attach')
-    pydevd._wait_for_attach()  # Should promptly return (already connected).
-
-    a = 20  # Break 2
-
-    pydevd._wait_for_attach()  # As we disconnected on the 2nd break, this one should wait until a new configurationDone.
-
-    a = 20  # Break 3
-
-    while a == 20:  # Pause 1
-        # The debugger should disconnect/reconnect, pause and then change 'a' to another value.
-        time.sleep(1 / 20.)  # Pause 2
-
-    print('TEST SUCEEDED!')
+    # We want to call _enable_attach inside an import to make sure that it works properly that way.
+    import _debugger_case_wait_for_attach_impl
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach_impl.py b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach_impl.py
new file mode 100644
--- /dev/null
+++ b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach_impl.py
@@ -0,0 +1,49 @@
+import os
+import sys
+import time
+port = int(sys.argv[1])
+root_dirname = os.path.dirname(os.path.dirname(__file__))
+
+if root_dirname not in sys.path:
+    sys.path.append(root_dirname)
+
+import pydevd
+try:
+    pydevd._wait_for_attach()  # Cannot be called before _enable_attach.
+except AssertionError:
+    pass
+else:
+    raise AssertionError('Expected _wait_for_attach to raise exception.')
+
+assert sys.gettrace() is None
+print('enable attach to port: %s' % (port,))
+pydevd._enable_attach(('127.0.0.1', port))
+pydevd._enable_attach(('127.0.0.1', port))  # no-op in practice
+
+try:
+    pydevd._enable_attach(('127.0.0.1', port + 15))  # different port: raise error.
+except AssertionError:
+    pass
+else:
+    raise AssertionError('Expected _enable_attach to raise exception (because it is already hearing in another port).')
+
+assert pydevd.get_global_debugger() is not None
+assert sys.gettrace() is not None
+
+a = 10  # Break 1
+print('wait for attach')
+pydevd._wait_for_attach()
+print('finished wait for attach')
+pydevd._wait_for_attach()  # Should promptly return (already connected).
+
+a = 20  # Break 2
+
+pydevd._wait_for_attach()  # As we disconnected on the 2nd break, this one should wait until a new configurationDone.
+
+a = 20  # Break 3
+
+while a == 20:  # Pause 1
+    # The debugger should disconnect/reconnect, pause and then change 'a' to another value.
+    time.sleep(1 / 20.)  # Pause 2
+
+print('TEST SUCEEDED!')
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
@@ -2426,6 +2426,7 @@ def check_process_event(json_facade, start_method):
         assert next(iter(process_events)).body.startMethod == start_method
 
     with case_setup_remote_attach_to.test_file('_debugger_case_wait_for_attach.py', host_port[1]) as writer:
+        writer.TEST_FILE = debugger_unittest._get_debugger_test_file('_debugger_case_wait_for_attach_impl.py')
         time.sleep(.5)  # Give some time for it to pass the first breakpoint and wait in 'wait_for_attach'.
         writer.start_socket_client(*host_port)