Repository: microsoft/debugpy

Issue:
Hide debugger threads from being returned in the threading module
This code should go to end but in VS Code does not. In others IDE this run to the end.
Is this BUG or? 

Code is:

```
import logging
import random
import threading
import time

logging.basicConfig(
    level=logging.DEBUG,
    format='(%(threadName)-10s) %(message)s',
)

class Counter(object):
    def __init__(self, start=0):
        self.lock = threading.Lock()
        self.value = start
    def increment(self):
        logging.debug('Waiting for lock')
        self.lock.acquire()
        try:
            logging.debug('Acquired lock')
            self.value = self.value + 1
        finally:
            logging.debug("Releasing")
            self.lock.release()

def worker(c):
    print('Entering worker...')
    for i in range(2):
        print('worker loop:', i)
        pause = random.random()
        logging.debug('Sleeping %0.02f', pause)
        time.sleep(pause)
        c.increment()
    logging.debug('Done')

counter = Counter()
for i in range(2):
    t = threading.Thread(target=worker, args=(counter,))
    t.start()

logging.debug('Waiting for worker threads')
main_thread = threading.currentThread()

try:
    for t in threading.enumerate():
        if t is not main_thread:
            t.join()
except Exception as e:
    print('exception: ', e)
finally:
    print('This is test')

logging.debug('Counter: %d', counter.value)

logging.debug('End of script')
```

Discussion:

Patch:
diff --git a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py
index 9b01a651..3866e752 100644
--- a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py
+++ b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py
@@ -123,3 +123,6 @@ else:
     with VerifyShadowedImport('http.server') as verify_shadowed:
         import http.server as BaseHTTPServer;    verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler'])
 
+# If set, this is a version of the threading.enumerate that doesn't have the patching to remove the pydevd threads.
+# Note: as it can't be set during execution, don't import the name (import the module and access it through its name).
+pydevd_saved_threading_enumerate = None
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py
index da1b5386..848af188 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_code_to_source.py
@@ -7,7 +7,7 @@ Note: this is a work in progress / proof of concept / not ready to be used.
 
 import dis
 
-from _pydevd_bundle.pydevd_collect_bytecode_info import _iter_instructions
+from _pydevd_bundle.pydevd_collect_bytecode_info import iter_instructions
 from _pydevd_bundle.pydevd_constants import dict_iter_items, IS_PY2
 from _pydev_bundle import pydev_log
 import sys
@@ -536,7 +536,7 @@ class _PyCodeToSource(object):
             memo = {}
         self.memo = memo
         self.co = co
-        self.instructions = list(_iter_instructions(co))
+        self.instructions = list(iter_instructions(co))
         self.stack = _Stack()
         self.writer = _Writer()
 
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py
index aa577a63..39da7150 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py
@@ -141,7 +141,7 @@ def _iter_as_bytecode_as_instructions_py2(co):
                 yield _Instruction(curr_op_name, op, _get_line(op_offset_to_line, initial_bytecode_offset, 0), oparg, is_jump_target, initial_bytecode_offset, str(oparg))
 
 
-def _iter_instructions(co):
+def iter_instructions(co):
     if sys.version_info[0] < 3:
         iter_in = _iter_as_bytecode_as_instructions_py2(co)
     else:
@@ -168,7 +168,7 @@ def collect_return_info(co, use_func_first_line=False):
 
     lst = []
     op_offset_to_line = dict(dis.findlinestarts(co))
-    for instruction in _iter_instructions(co):
+    for instruction in iter_instructions(co):
         curr_op_name = instruction.opname
         if curr_op_name == 'RETURN_VALUE':
             lst.append(ReturnInfo(_get_line(op_offset_to_line, instruction.offset, firstlineno, search=True)))
@@ -192,7 +192,7 @@ if sys.version_info[:2] < (3, 5):
 
         op_offset_to_line = dict(dis.findlinestarts(co))
 
-        for instruction in _iter_instructions(co):
+        for instruction in iter_instructions(co):
             curr_op_name = instruction.opname
 
             if curr_op_name in ('SETUP_EXCEPT', 'SETUP_FINALLY'):
@@ -351,7 +351,7 @@ if (3, 5) <= sys.version_info[:2] <= (3, 9):
 
         offset_to_instruction_idx = {}
 
-        instructions = list(_iter_instructions(co))
+        instructions = list(iter_instructions(co))
 
         for i, instruction in enumerate(instructions):
             offset_to_instruction_idx[instruction.offset] = i
@@ -471,7 +471,7 @@ if sys.version_info[:2] >= (3, 10):
 
         offset_to_instruction_idx = {}
 
-        instructions = list(_iter_instructions(co))
+        instructions = list(iter_instructions(co))
 
         for i, instruction in enumerate(instructions):
             offset_to_instruction_idx[instruction.offset] = i
@@ -656,7 +656,7 @@ class _Disassembler(object):
         self.co = co
         self.firstlineno = firstlineno
         self.level = level
-        self.instructions = list(_iter_instructions(co))
+        self.instructions = list(iter_instructions(co))
         op_offset_to_line = self.op_offset_to_line = dict(dis.findlinestarts(co))
 
         # Update offsets so that all offsets have the line index (and update it based on
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
index c9a32c5b..977f729b 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
@@ -317,6 +317,9 @@ PYDEVD_UNBLOCK_THREADS_TIMEOUT = as_float_in_env('PYDEVD_UNBLOCK_THREADS_TIMEOUT
 # on how the thread interruption works (there are some caveats related to it).
 PYDEVD_INTERRUPT_THREAD_TIMEOUT = as_float_in_env('PYDEVD_INTERRUPT_THREAD_TIMEOUT', -1)
 
+# If PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS is set to False, the patching to hide pydevd threads won't be applied.
+PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS = os.getenv('PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS', 'true').lower() in ENV_TRUE_LOWER_VALUES
+
 EXCEPTION_TYPE_UNHANDLED = 'UNHANDLED'
 EXCEPTION_TYPE_USER_UNHANDLED = 'USER_UNHANDLED'
 EXCEPTION_TYPE_HANDLED = 'HANDLED'
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py
index 1eb40311..6599836a 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py
@@ -1,11 +1,14 @@
 from _pydev_imps._pydev_saved_modules import threading
+from _pydev_imps import _pydev_saved_modules
 from _pydevd_bundle.pydevd_utils import notify_about_gevent_if_needed
 import weakref
-from _pydevd_bundle.pydevd_constants import IS_JYTHON
+from _pydevd_bundle.pydevd_constants import IS_JYTHON, IS_IRONPYTHON, \
+    PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS
 from _pydev_bundle.pydev_log import exception as pydev_log_exception
 import sys
 from _pydev_bundle import pydev_log
 import pydevd_tracing
+from _pydevd_bundle.pydevd_collect_bytecode_info import iter_instructions
 
 if IS_JYTHON:
     import org.python.core as JyCore  # @UnresolvedImport
@@ -67,7 +70,111 @@ class PyDBDaemonThread(threading.Thread):
             pydevd_tracing.SetTrace(None)  # no debugging on this thread
 
 
+def _collect_load_names(func):
+    found_load_names = set()
+    for instruction in iter_instructions(func.__code__):
+        if instruction.opname in ('LOAD_GLOBAL', 'LOAD_ATTR', 'LOAD_METHOD'):
+            found_load_names.add(instruction.argrepr)
+    return found_load_names
+
+
+def _patch_threading_to_hide_pydevd_threads():
+    '''
+    Patches the needed functions on the `threading` module so that the pydevd threads are hidden.
+
+    Note that we patch the functions __code__ to avoid issues if some code had already imported those
+    variables prior to the patching.
+    '''
+    found_load_names = _collect_load_names(threading.enumerate)
+    # i.e.: we'll only apply the patching if the function seems to be what we expect.
+
+    new_threading_enumerate = None
+
+    if found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values', 'list')):
+        pydev_log.debug('Applying patching to hide pydevd threads (Py3 version).')
+
+        def new_threading_enumerate():
+            with _active_limbo_lock:
+                ret = list(_active.values()) + list(_limbo.values())
+
+            return [t for t in ret if not getattr(t, 'is_pydev_daemon_thread', False)]
+
+    elif found_load_names == set(('_active_limbo_lock', '_limbo', '_active', 'values')):
+        pydev_log.debug('Applying patching to hide pydevd threads (Py2 version).')
+
+        def new_threading_enumerate():
+            with _active_limbo_lock:
+                ret = _active.values() + _limbo.values()
+
+            return [t for t in ret if not getattr(t, 'is_pydev_daemon_thread', False)]
+
+    else:
+        pydev_log.info('Unable to hide pydevd threads. Found names in threading.enumerate: %s', found_load_names)
+
+    if new_threading_enumerate is not None:
+
+        def pydevd_saved_threading_enumerate():
+            with threading._active_limbo_lock:
+                return list(threading._active.values()) + list(threading._limbo.values())
+
+        _pydev_saved_modules.pydevd_saved_threading_enumerate = pydevd_saved_threading_enumerate
+
+        threading.enumerate.__code__ = new_threading_enumerate.__code__
+
+        # We also need to patch the active count (to match what we have in the enumerate).
+        def new_active_count():
+            # Note: as this will be executed in the `threading` module, `enumerate` will
+            # actually be threading.enumerate.
+            return len(enumerate())
+
+        threading.active_count.__code__ = new_active_count.__code__
+
+        # When shutting down, Python (on some versions) may do something as:
+        #
+        # def _pickSomeNonDaemonThread():
+        #     for t in enumerate():
+        #         if not t.daemon and t.is_alive():
+        #             return t
+        #     return None
+        #
+        # But in this particular case, we do want threads with `is_pydev_daemon_thread` to appear
+        # explicitly due to the pydevd `CheckAliveThread` (because we want the shutdown to wait on it).
+        # So, it can't rely on the `enumerate` for that anymore as it's patched to not return pydevd threads.
+        if hasattr(threading, '_pickSomeNonDaemonThread'):
+
+            def new_pick_some_non_daemon_thread():
+                with _active_limbo_lock:
+                    # Ok for py2 and py3.
+                    threads = list(_active.values()) + list(_limbo.values())
+
+                for t in threads:
+                    if not t.daemon and t.is_alive():
+                        return t
+                return None
+
+            threading._pickSomeNonDaemonThread.__code__ = new_pick_some_non_daemon_thread.__code__
+
+
+_patched_threading_to_hide_pydevd_threads = False
+
+
 def mark_as_pydevd_daemon_thread(thread):
+    if not IS_JYTHON and not IS_IRONPYTHON and PYDEVD_APPLY_PATCHING_TO_HIDE_PYDEVD_THREADS:
+        global _patched_threading_to_hide_pydevd_threads
+        if not _patched_threading_to_hide_pydevd_threads:
+            # When we mark the first thread as a pydevd daemon thread, we also change the threading
+            # functions to hide pydevd threads.
+            # Note: we don't just "hide" the pydevd threads from the threading module by not using it
+            # (i.e.: just using the `thread.start_new_thread` instead of `threading.Thread`)
+            # because there's 1 thread (the `CheckAliveThread`) which is a pydevd thread but
+            # isn't really a daemon thread (so, we need CPython to wait on it for shutdown,
+            # in which case it needs to be in `threading` and the patching would be needed anyways).
+            _patched_threading_to_hide_pydevd_threads = True
+            try:
+                _patch_threading_to_hide_pydevd_threads()
+            except:
+                pydev_log.exception('Error applying patching to hide pydevd threads.')
+
     thread.pydev_do_not_trace = True
     thread.is_pydev_daemon_thread = True
     thread.daemon = True
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py
index 5b3d539d..6e4af794 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py
@@ -3,6 +3,7 @@ import traceback
 import warnings
 from _pydev_bundle import pydev_log
 from _pydev_imps._pydev_saved_modules import thread
+from _pydev_imps import _pydev_saved_modules
 import signal
 import os
 import ctypes
@@ -177,7 +178,11 @@ def dump_threads(stream=None, show_pydevd_threads=True):
         stream = sys.stderr
     thread_id_to_name_and_is_pydevd_thread = {}
     try:
-        for t in threading.enumerate():
+        threading_enumerate = _pydev_saved_modules.pydevd_saved_threading_enumerate
+        if threading_enumerate is None:
+            threading_enumerate = threading.enumerate
+
+        for t in threading_enumerate():
             is_pydevd_thread = getattr(t, 'is_pydev_daemon_thread', False)
             thread_id_to_name_and_is_pydevd_thread[t.ident] = (
                 '%s  (daemon: %s, pydevd thread: %s)' % (t.name, t.daemon, is_pydevd_thread),
@@ -297,6 +302,7 @@ def hasattr_checked(obj, name):
     else:
         return True
 
+
 def getattr_checked(obj, name):
     try:
         return getattr(obj, name)


Test patch:
diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py
index cba32237..8190d356 100644
--- a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py
+++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py
@@ -4372,6 +4372,33 @@ def test_debugger_shadowed_imports(case_setup, tmpdir, module_name_and_content):
 
     assert ('the module "%s" could not be imported because it is shadowed by:' % (module_name.split('.')[0])) in writer.get_stderr()
 
+
+def test_debugger_hide_pydevd_threads(case_setup, pyfile):
+
+    @pyfile
+    def target_file():
+        import threading
+        from _pydevd_bundle import pydevd_constants
+        found_pydevd_thread = False
+        for t in threading.enumerate():
+            if getattr(t, 'is_pydev_daemon_thread', False):
+                found_pydevd_thread = True
+
+        if pydevd_constants.IS_CPYTHON:
+            assert not found_pydevd_thread
+        else:
+            assert found_pydevd_thread
+        print('TEST SUCEEDED')
+
+    with case_setup.test_file(target_file) as writer:
+        line = writer.get_line_index_with_content('TEST SUCEEDED')
+        writer.write_add_breakpoint(line)
+        writer.write_make_initial_run()
+
+        hit = writer.wait_for_breakpoint_hit(line=line)
+        writer.write_run_thread(hit.thread_id)
+        writer.finished_ok = True
+
 # Jython needs some vars to be set locally.
 # set JAVA_HOME=c:\bin\jdk1.8.0_172
 # set PATH=%PATH%;C:\bin\jython2.7.0\bin
diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py b/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py
index ba21361d..4f3e65f9 100644
--- a/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py
+++ b/src/debugpy/_vendored/pydevd/tests_python/test_utilities.py
@@ -3,7 +3,7 @@ import threading
 from _pydevd_bundle.pydevd_utils import convert_dap_log_message_to_expression
 from tests_python.debug_constants import IS_PY26, IS_PY3K, TEST_GEVENT, IS_CPYTHON
 import sys
-from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY2, IS_PYPY
+from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY2, IS_PYPY, IS_JYTHON
 import pytest
 import os
 import codecs
@@ -424,7 +424,7 @@ def test_find_main_thread_id():
     )
 
 
-@pytest.mark.skipif(not IS_WINDOWS, reason='Windows-only test.')
+@pytest.mark.skipif(not IS_WINDOWS or IS_JYTHON, reason='Windows-only test.')
 def test_get_ppid():
     from _pydevd_bundle.pydevd_api import PyDevdAPI
     api = PyDevdAPI()
@@ -522,3 +522,38 @@ def test_get_smart_step_into_variant_from_frame_offset():
     assert get_smart_step_into_variant_from_frame_offset(2, variants) is variants[0]
     assert get_smart_step_into_variant_from_frame_offset(3, variants) is variants[1]
     assert get_smart_step_into_variant_from_frame_offset(4, variants) is variants[1]
+
+
+def test_threading_hide_pydevd():
+
+    class T(threading.Thread):
+
+        def __init__(self, is_pydev_daemon_thread):
+            from _pydevd_bundle.pydevd_daemon_thread import mark_as_pydevd_daemon_thread
+            threading.Thread.__init__(self)
+            if is_pydev_daemon_thread:
+                mark_as_pydevd_daemon_thread(self)
+            else:
+                self.daemon = True
+            self.event = threading.Event()
+
+        def run(self):
+            self.event.wait(10)
+
+    current_count = threading.active_count()
+    t0 = T(True)
+    t1 = T(False)
+    t0.start()
+    t1.start()
+
+    # i.e.: the patching doesn't work for other implementations.
+    if IS_CPYTHON:
+        assert threading.active_count() == current_count + 1
+        assert t0 not in threading.enumerate()
+    else:
+        assert threading.active_count() == current_count + 2
+        assert t0 in threading.enumerate()
+
+    assert t1 in threading.enumerate()
+    t0.event.set()
+    t1.event.set()