Repository: microsoft/ptvsd

Issue:
Some Mac site-package locations are not treated as library by the debugger
For example, this location is not treated as library by `pydevd` in Python 3.7:
`/Users/vsts/Library/Python/3.7/lib/python/site-packages`

NOTE: To reproduce this issue with ptvsd. Remove the line in `wrapper.py` where we set project root (look for CMD_SET_PROJECT_ROOTS). Then run the pollowing test on Mac python 3.7 
```
python3 -m tests -v tests.system_tests.test_web_frameworks.FlaskLaunchTests.test_with_unhandled_exceptions
```

For reference, these are the sys.path values from our CI environment:
```
Travis (linux):
/home/travis/build/karthiknadig/ptvsd/tests/resources/system_tests/test_web_frameworks/flask/launch
/home/travis/build/karthiknadig/ptvsd
/home/travis/virtualenv/python3.7-dev/lib/python37.zip
/home/travis/virtualenv/python3.7-dev/lib/python3.7
/home/travis/virtualenv/python3.7-dev/lib/python3.7/lib-dynload
/opt/python/3.7-dev/lib/python3.7
/home/travis/virtualenv/python3.7-dev/lib/python3.7/site-packages

VSTS (windows):
D:\a\1\s\tests\resources\system_tests\test_web_frameworks\flask\launch
D:\a\1\s;C:\hostedtoolcache\windows\Python\3.7.0\x64\python37.zip
C:\hostedtoolcache\windows\Python\3.7.0\x64\DLLs
C:\hostedtoolcache\windows\Python\3.7.0\x64\lib
C:\hostedtoolcache\windows\Python\3.7.0\x64
C:\hostedtoolcache\windows\Python\3.7.0\x64\lib\site-packages

VSTS (linux):
/home/vsts/work/1/s/tests/resources/system_tests/test_web_frameworks/flask/launch
/home/vsts/work/1/s
/opt/hostedtoolcache/Python/3.7.0/x64/lib/python37.zip
/opt/hostedtoolcache/Python/3.7.0/x64/lib/python3.7
/opt/hostedtoolcache/Python/3.7.0/x64/lib/python3.7/lib-dynload
/opt/hostedtoolcache/Python/3.7.0/x64/lib/python3.7/site-packages

VSTS (Mac 3.7):
/Users/vsts/agent/2.140.0/work/1/s/tests/resources/system_tests/test_web_frameworks/flask/attach
/Users/vsts/agent/2.140.0/work/1/s;/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python37.zip
/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7
/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/lib-dynload
/Users/vsts/Library/Python/3.7/lib/python/site-packages
/usr/local/lib/python3.7/site-packages
```




Discussion:
Strange... in my tests it seems that `/Users/vsts/Library/Python/3.7/lib/python/site-packages` does appear as a site package for `pydevd` (although `/usr/local/lib/python3.7/site-packages` doesn't and I still didn't discover how to get this entry).

Anyways, just giving some info, will investigate further.

Resolving patch:
diff --git a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py
--- a/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py
+++ b/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py
@@ -15,6 +15,7 @@
 from _pydev_bundle import pydev_log
 from _pydev_imps._pydev_saved_modules import threading
 
+
 def _normpath(filename):
     return pydevd_file_utils.get_abs_path_real_path_and_base_from_file(filename)[0]
 
@@ -148,7 +149,7 @@ def _convert_to_str_and_clear_empty(roots):
             root if not isinstance(root, unicode) else root.encode(sys.getfilesystemencoding())
             for root in roots
         ]
-    
+
     new_roots = []
     for root in roots:
         assert isinstance(root, str), '%s not str (found: %s)' % (root, type(root))
@@ -217,7 +218,12 @@ def _get_default_library_roots():
                 roots.append(site_path)
         else:
             roots.append(site_paths)
-    return roots
+
+    for path in sys.path:
+        if os.path.exists(path) and os.path.basename(path) == 'site-packages':
+            roots.append(path)
+
+    return sorted(set(roots))
 
 
 # --- Project roots
@@ -235,7 +241,7 @@ def set_library_roots(roots):
     roots = _set_roots(roots, _LIBRARY_ROOTS_CACHE)
     pydev_log.debug("LIBRARY_ROOTS %s\n" % roots)
 
-    
+
 def _get_library_roots(library_roots_cache=_LIBRARY_ROOTS_CACHE):
     return _get_roots(library_roots_cache, 'LIBRARY_ROOTS', set_library_roots, _get_default_library_roots)
 
@@ -249,7 +255,7 @@ def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_
         original_filename = filename
         if not filename.endswith('>'):
             filename = _normpath(filename)
-        
+
         found_in_project = []
         for root in project_roots:
             if root and filename.startswith(root):
@@ -269,7 +275,7 @@ def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_
                 in_project = False
             else:
                 in_project = not found_in_library
-        else:                
+        else:
             in_project = False
             if found_in_project:
                 if not found_in_library:
@@ -278,7 +284,7 @@ def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_
                     # Found in both, let's see which one has the bigger path matched.
                     if max(len(x) for x in found_in_project) > max(len(x) for x in found_in_library):
                         in_project = True
-                    
+
         filename_to_in_scope_cache[original_filename] = in_project
         return in_project
 
@@ -333,7 +339,7 @@ def dump_threads(stream=None):
                 t.name, t.daemon, getattr(t, 'is_pydev_daemon_thread', False))
     except:
         pass
-    
+
     from _pydevd_bundle.pydevd_additional_thread_info_regular import _current_frames
 
     stream.write('===============================================================================\n')


Test patch:
diff --git a/ptvsd/_vendored/pydevd/tests_python/test_in_project_roots.py b/ptvsd/_vendored/pydevd/tests_python/test_in_project_roots.py
--- a/ptvsd/_vendored/pydevd/tests_python/test_in_project_roots.py
+++ b/ptvsd/_vendored/pydevd/tests_python/test_in_project_roots.py
@@ -1,20 +1,21 @@
 def test_in_project_roots(tmpdir):
     from _pydevd_bundle import pydevd_utils
     import os.path
+    import sys
     assert pydevd_utils._get_library_roots() == [
         os.path.normcase(x) for x in pydevd_utils._get_default_library_roots()]
-    
+
     site_packages = tmpdir.mkdir('site-packages')
     project_dir = tmpdir.mkdir('project')
-    
+
     project_dir_inside_site_packages = str(site_packages.mkdir('project'))
     site_packages_inside_project_dir = str(project_dir.mkdir('site-packages'))
-    
+
     # Convert from pytest paths to str.
     site_packages = str(site_packages)
     project_dir = str(project_dir)
     tmpdir = str(tmpdir)
-    
+
     # Test permutations of project dir inside site packages and vice-versa.
     pydevd_utils.set_project_roots([project_dir, project_dir_inside_site_packages])
     pydevd_utils.set_library_roots([site_packages, site_packages_inside_project_dir])
@@ -28,15 +29,15 @@ def test_in_project_roots(tmpdir):
     ]
     for (check_path, find) in check[:]:
         check.append((os.path.join(check_path, 'a.py'), find))
-        
+
     for check_path, find in check:
         assert pydevd_utils.in_project_roots(check_path) == find
 
     pydevd_utils.set_project_roots([])
     pydevd_utils.set_library_roots([site_packages, site_packages_inside_project_dir])
-    
+
     # If the IDE did not set the project roots, consider anything not in the site
-    # packages as being in a project root (i.e.: we can calculate default values for 
+    # packages as being in a project root (i.e.: we can calculate default values for
     # site-packages but not for project roots).
     check = [
         (tmpdir, True),
@@ -46,6 +47,19 @@ def test_in_project_roots(tmpdir):
         (project_dir_inside_site_packages, False),
         (os.path.join(tmpdir, '<foo>'), False),
     ]
-    
+
     for check_path, find in check:
         assert pydevd_utils.in_project_roots(check_path) == find
+
+    sys.path.append(str(site_packages))
+    try:
+        default_library_roots = pydevd_utils._get_default_library_roots()
+        assert len(set(default_library_roots)) == len(default_library_roots), \
+            'Duplicated library roots found in: %s' % (default_library_roots,)
+    
+        assert str(site_packages) in default_library_roots
+        for path in sys.path:
+            if os.path.exists(path) and path.endswith('site-packages'):
+                assert path in default_library_roots
+    finally:
+        sys.path.remove(str(site_packages))