Repository: microsoft/ptvsd

Issue:
Normalize paths using `glob` or similar on Windows
@int19h Please can we implement the less than ideal solution of using the `glob` or similar approach to normalize paths on Windows.
We'd like to ensure we have some form of a fix, as we're not too certain when we'd have a fix from VSC.
Yes, we're aware this won't fix all scenarios.



Discussion:
I'll take a look at this.

Resolving patch:
diff --git a/src/ptvsd/_vendored/pydevd/pydevd_file_utils.py b/src/ptvsd/_vendored/pydevd/pydevd_file_utils.py
--- a/src/ptvsd/_vendored/pydevd/pydevd_file_utils.py
+++ b/src/ptvsd/_vendored/pydevd/pydevd_file_utils.py
@@ -59,6 +59,11 @@
 exists = os.path.exists
 join = os.path.join
 
+try:
+    FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError  # noqa
+
 try:
     rPath = os.path.realpath  # @UndefinedVariable
 except:
@@ -132,13 +137,78 @@ def _convert_to_short_pathname(filename):
                 filename = filename.encode(getfilesystemencoding())
             return filename
 
+        # Note that we have a cache for previous list dirs... the only case where this may be an
+        # issue is if the user actually changes the case of an existing file on windows while
+        # the debugger is executing (as this seems very unlikely and the cache can save a
+        # reasonable time -- especially on mapped drives -- it seems nice to have it).
+        _listdir_cache = {}
+
+        def _resolve_listing(resolved, iter_parts, cache=_listdir_cache):
+            while True:  # Note: while True to make iterative and not recursive
+                try:
+                    resolve_lowercase = next(iter_parts)  # must be lowercase already
+                except StopIteration:
+                    return resolved
+
+                resolved_lower = resolved.lower()
+
+                resolved_joined = cache.get((resolved_lower, resolve_lowercase))
+                if resolved_joined is None:
+                    dir_contents = cache.get(resolved_lower)
+                    if dir_contents is None:
+                        dir_contents = cache[resolved_lower] = os.listdir(resolved)
+
+                    for filename in dir_contents:
+                        if filename.lower() == resolve_lowercase:
+                            resolved_joined = os.path.join(resolved, filename)
+                            cache[(resolved_lower, resolve_lowercase)] = resolved_joined
+                            break
+                    else:
+                        raise FileNotFoundError('Unable to find: %s in %s' % (
+                            resolve_lowercase, resolved))
+
+                resolved = resolved_joined
+
         def _get_path_with_real_case(filename):
-            ret = convert_to_long_pathname(convert_to_short_pathname(filename))
-            # This doesn't handle the drive letter properly (it'll be unchanged).
-            # Make sure the drive letter is always uppercase.
-            if len(ret) > 1 and ret[1] == ':' and ret[0].islower():
-                return ret[0].upper() + ret[1:]
-            return ret
+            # Note: this previously made:
+            # convert_to_long_pathname(convert_to_short_pathname(filename))
+            # but this is no longer done because we can't rely on getting the shortname
+            # consistently (there are settings to disable it on Windows).
+            # So, using approach which resolves by listing the dir.
+
+            if IS_PY2 and isinstance(filename, unicode):  # noqa
+                filename = filename.encode(getfilesystemencoding())
+
+            if '~' in filename:
+                filename = convert_to_long_pathname(filename)
+
+            if filename.startswith('<') or not os.path.exists(filename):
+                return filename  # Not much we can do.
+
+            drive, parts = os.path.splitdrive(os.path.normpath(filename))
+            drive = drive.upper()
+            while parts.startswith(os.path.sep):
+                parts = parts[1:]
+                drive += os.path.sep
+            parts = parts.lower().split(os.path.sep)
+
+            try:
+                return _resolve_listing(drive, iter(parts))
+            except FileNotFoundError:
+                _listdir_cache.clear()
+                # Retry once after clearing the cache we have.
+                try:
+                    return _resolve_listing(drive, iter(parts))
+                except FileNotFoundError:
+                    if os.path.exists(filename):
+                        # This is really strange, ask the user to report as error.
+                        sys.stderr.write('\npydev debugger: critical: unable to get real case for file. Details:\n'
+                                         'filename: %s\ndrive: %s\nparts: %s\n'
+                                         '(please create a ticket in the tracker to address this).\n\n' % (
+                                             filename, drive, parts))
+                        traceback.print_exc()
+                    # Don't fail, just return the original file passed.
+                    return filename
 
         # Check that it actually works
         _get_path_with_real_case(__file__)


Test patch:
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_convert_utilities.py b/src/ptvsd/_vendored/pydevd/tests_python/test_convert_utilities.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/test_convert_utilities.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/test_convert_utilities.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 import os.path
-from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_JYTHON
+from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_JYTHON, IS_PY2
+from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding
 
 
 def test_convert_utilities(tmpdir):
@@ -24,19 +25,43 @@ def test_convert_utilities(tmpdir):
             assert stream.read() == 'test'
 
         assert '~' not in normalized
-        if not IS_JYTHON:
-            assert '~' in pydevd_file_utils.convert_to_short_pathname(normalized)
 
-        real_case = pydevd_file_utils.get_path_with_real_case(normalized)
-        assert isinstance(real_case, str)  # bytes on py2, unicode on py3
-        # Note test_dir itself cannot be compared with because pytest may
-        # have passed the case normalized.
-        assert real_case.endswith("Test_Convert_Utilities")
+        for i in range(3):  # Check if cache is ok.
+
+            if i == 2:
+                pydevd_file_utils._listdir_cache.clear()
+
+            assert pydevd_file_utils.get_path_with_real_case('<does not EXIST>') == '<does not EXIST>'
+            real_case = pydevd_file_utils.get_path_with_real_case(normalized)
+            assert isinstance(real_case, str)  # bytes on py2, unicode on py3
+            # Note test_dir itself cannot be compared with because pytest may
+            # have passed the case normalized.
+            assert real_case.endswith("Test_Convert_Utilities")
+
+            if i == 2:
+                # Check that we have the expected paths in the cache.
+                assert pydevd_file_utils._listdir_cache[os.path.dirname(normalized).lower()] == ['Test_Convert_Utilities']
+                assert pydevd_file_utils._listdir_cache[(os.path.dirname(normalized).lower(), 'Test_Convert_Utilities'.lower())] == real_case
+
+            if IS_PY2:
+                # Test with unicode in python 2 too.
+                real_case = pydevd_file_utils.get_path_with_real_case(normalized.decode(
+                    getfilesystemencoding()))
+                assert isinstance(real_case, str)  # bytes on py2, unicode on py3
+                # Note test_dir itself cannot be compared with because pytest may
+                # have passed the case normalized.
+                assert real_case.endswith("Test_Convert_Utilities")
+
+        # Check that it works with a shortened path.
+        shortened = pydevd_file_utils.convert_to_short_pathname(normalized)
+        assert '~' in shortened
+        with_real_case = pydevd_file_utils.get_path_with_real_case(shortened)
+        assert with_real_case.endswith('Test_Convert_Utilities')
+        assert '~' not in with_real_case
 
     else:
         # On other platforms, nothing should change
         assert pydevd_file_utils.normcase(test_dir) == test_dir
-        assert pydevd_file_utils.convert_to_short_pathname(test_dir) == test_dir
         assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir