Repository: microsoft/ptvsd

Issue:
Path with non-ASCII characters won't hit breakpoints
* Create a file named `汉语a2.py` with the code:
```python
print("Hello world")
```
* Add break point to the first line
* Start debugging in **Python 2.7**
* Debugger does not stop at the breakpoint


Here's the json comms:
<section><details>
```js
1:24:05 PM, 5/16/2018
Started @ Wed May 16 2018 13:24:05 GMT-0400 (EDT)
From Client:
Content-Length: 336

{"command":"initialize","arguments":{"clientID":"vscode","clientName":"Visual Studio Code - Insiders","adapterID":"pythonExperimental","pathFormat":"path","linesStartAt1":true,"columnsStartAt1":true,"supportsVariableType":true,"supportsVariablePaging":true,"supportsRunInTerminalRequest":true,"locale":"en-us"},"type":"request","seq":1}
To Client:
Content-Length: 621

{"seq":1,"type":"response","request_seq":1,"command":"initialize","success":true,"body":{"supportsExceptionInfoRequest":true,"supportsConfigurationDoneRequest":true,"supportsConditionalBreakpoints":true,"supportsSetVariable":true,"supportsExceptionOptions":true,"supportsEvaluateForHovers":true,"supportsModulesRequest":true,"supportsValueFormattingOptions":true,"supportsHitConditionalBreakpoints":true,"supportsSetExpression":true,"supportsLogPoints":true,"exceptionBreakpointFilters":[{"filter":"raised","label":"Raised Exceptions","default":false},{"filter":"uncaught","label":"Uncaught Exceptions","default":true}]}}
From Client:
Content-Length: 688

{"command":"launch","arguments":{"name":"Python Experimental: Terminal (integrated)","type":"pythonExperimental","request":"launch","pythonPath":"/usr/bin/python2.7","program":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/汉语a2.py","console":"integratedTerminal","envFile":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/.myenvx","logToFile":true,"logLevel":"info","cwd":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo","stopOnEntry":false,"internalConsoleOptions":"neverOpen","debugOptions":["RedirectOutput"],"__sessionId":"8f978268-89f2-4275-93e0-a89d1b7c03ae"},"type":"request","seq":2}
To Client:
Content-Length: 541

{"command":"runInTerminal","arguments":{"kind":"integrated","title":"Python Debug Console","cwd":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo","args":["/usr/bin/python2.7","-m","ptvsd","--host","localhost","--port","54344","/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/汉语a2.py"],"env":{"PYTHONIOENCODING":"UTF-8","PYTHONUNBUFFERED":"1","PYTHONPATH":"/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd"}},"type":"request","seq":2}
From Client:
Content-Length: 94

{"type":"response","seq":3,"command":"runInTerminal","request_seq":2,"success":true,"body":{}}
To Client:
Content-Length: 688


To Client:
{"command":"launch","arguments":{"name":"Python Experimental: Terminal (integrated)","type":"pythonExperimental","request":"launch","pythonPath":"/usr/bin/python2.7","program":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/汉语a2.py","console":"integratedTerminal","envFile":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/.myenvx","logToFile":true,"logLevel":"info","cwd":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo","stopOnEntry":false,"internalConsoleOptions":"neverOpen","debugOptions":["RedirectOutput"],"__sessionId":"8f978268-89f2-4275-93e0-a89d1b7c03ae"},"type":"request","seq":2}
To Client:
Content-Length: 336


To Client:
{"command":"initialize","arguments":{"clientID":"vscode","clientName":"Visual Studio Code - Insiders","adapterID":"pythonExperimental","pathFormat":"path","linesStartAt1":true,"columnsStartAt1":true,"supportsVariableType":true,"supportsVariablePaging":true,"supportsRunInTerminalRequest":true,"locale":"en-us"},"type":"request","seq":1}
To Client:
Content-Length: 143

{"body": {"category": "telemetry", "output": "ptvsd", "data": {"version": "4.1.1a2+9.gbb25982"}}, "type": "event", "event": "output", "seq": 0}
To Client:
Content-Length: 681


To Client:
{"request_seq": 1, "body": {"supportsModulesRequest": true, "supportsConfigurationDoneRequest": true, "supportsHitConditionalBreakpoints": true, "supportsSetExpression": true, "supportsSetVariable": true, "exceptionBreakpointFilters": [{"filter": "raised", "default": false, "label": "Raised Exceptions"}, {"filter": "uncaught", "default": true, "label": "Uncaught Exceptions"}], "supportsValueFormattingOptions": true, "supportsExceptionInfoRequest": true, "supportsLogPoints": true, "supportsEvaluateForHovers": true, "supportsExceptionOptions": true, "supportsConditionalBreakpoints": true}, "seq": 1, "success": true, "command": "initialize", "message": "", "type": "response"}
To Client:
Content-Length: 63

{"body": {}, "type": "event", "event": "initialized", "seq": 2}
To Client:
Content-Length: 113


To Client:
{"request_seq": 2, "body": {}, "seq": 3, "success": true, "command": "launch", "message": "", "type": "response"}
From Client:
Content-Length: 269

{"command":"setBreakpoints","arguments":{"source":{"name":"compl1.py","path":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/compl1.py"},"lines":[4,6],"breakpoints":[{"line":4},{"line":6}],"sourceModified":false},"type":"request","seq":4}
From Client:
Content-Length: 250

{"command":"setBreakpoints","arguments":{"source":{"name":"so1.py","path":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/so1.py"},"lines":[4],"breakpoints":[{"line":4}],"sourceModified":false},"type":"request","seq":5}Content-Length: 260

{"command":"setBreakpoints","arguments":{"source":{"name":"汉语a2.py","path":"/Users/donjayamanne/Desktop/Development/PythonStuff/IssueRepos/aThrowAwayRepo/汉语a2.py"},"lines":[1],"breakpoints":[{"line":1}],"sourceModified":false},"type":"request","seq":6}
To Client:
Content-Length: 216


To Client:
{"request_seq": 4, "body": {"breakpoints": [{"line": 4, "verified": true, "id": 1}, {"line": 6, "verified": true, "id": 2}]}, "seq": 4, "success": true, "command": "setBreakpoints", "message": "", "type": "response"}
To Client:
Content-Length: 176

{"request_seq": 5, "body": {"breakpoints": [{"line": 4, "verified": true, "id": 3}]}, "seq": 5, "success": true, "command": "setBreakpoints", "message": "", "type": "response"}
To Client:
Content-Length: 990

{"request_seq": 6, "body": {}, "seq": 6, "success": false, "command": "setBreakpoints", "message": "Traceback (most recent call last):\n  File \"/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/ipcjson.py\", line 336, in on_request\n    target(request, args)\n  File \"/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/wrapper.py\", line 881, in f\n    fut = m(self, self.loop, *args, **kwargs)\n  File \"/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/futures.py\", line 140, in g\n    it = f(self, *args, **kwargs)\n  File \"/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/wrapper.py\", line 1739, in on_setBreakpoints\n    expression, hit_condition, is_logpoint)\nUnicodeEncodeError: 'ascii' codec can't encode characters in position 78-79: ordinal not in range(128)\n", "type": "response"}
From Client:
Content-Length: 56

{"command":"configurationDone","type":"request","seq":7}
From Client:
Content-Length: 46

{"command":"threads","type":"request","seq":8}
To Client:
Content-Length: 124


To Client:
{"request_seq": 7, "body": {}, "seq": 7, "success": true, "command": "configurationDone", "message": "", "type": "response"}
To Client:
Content-Length: 92

{"body": {"reason": "started", "threadId": 1}, "type": "event", "event": "thread", "seq": 8}
To Client:
Content-Length: 158

{"request_seq": 8, "body": {"threads": [{"id": 1, "name": "MainThread"}]}, "seq": 9, "success": true, "command": "threads", "message": "", "type": "response"}
From Client:
Content-Length: 46

{"command":"threads","type":"request","seq":9}
To Client:
Content-Length: 128


To Client:
{"request_seq": 9, "body": {"threads": []}, "seq": 10, "success": true, "command": "threads", "message": "", "type": "response"}
To Client:
Content-Length: 98

{"body": {"category": "stdout", "output": "ONE\n"}, "type": "event", "event": "output", "seq": 11}
To Client:
Content-Length: 72


onEventTerminated
To Client:
{"body": {"exitCode": 0}, "type": "event", "event": "exited", "seq": 12}Content-Length: 63

{"body": {}, "type": "event", "event": "terminated", "seq": 13}
From Client:
Content-Length: 80

{"command":"disconnect","arguments":{"restart":false},"type":"request","seq":10}
To Client:
Content-Length: 119


onResponseDisconnect
To Client:
{"request_seq": 10, "body": {}, "seq": 14, "success": true, "command": "disconnect", "message": "", "type": "response"}
check and shutdown
shutdown
Shutting down debug session
disposing
check and shutdown
check and shutdown
```
</details></section>


Discussion:
@brettcannon Please could you assign a milestone for this issue.
Seems to work for me on windows py 3.6:
![image](https://user-images.githubusercontent.com/3840081/40164362-2b952e14-596e-11e8-9561-29804640834f.png)

Python 2.7 has some other issue:
```
Traceback (most recent call last):
  File "C:\Python27\lib\runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "C:\Python27\lib\runpy.py", line 72, in _run_code
    exec code in run_globals
  File "C:\GIT\ptvsd\ptvsd\__main__.py", line 202, in <module>
    main(args.address, args.name, args.kind, extra, nodebug=args.nodebug)
  File "C:\GIT\ptvsd\ptvsd\__main__.py", line 197, in main
    debug_main(addr, name, kind, *extra)
  File "C:\GIT\ptvsd\ptvsd\_main.py", line 20, in debug_main
    run_file(address, name, *extra, **kwargs)
  File "C:\GIT\ptvsd\ptvsd\_main.py", line 47, in run_file
    run(argv, addr, **kwargs)
  File "C:\GIT\ptvsd\ptvsd\_main.py", line 98, in _run
    _pydevd.main()
  File "C:\GIT\ptvsd\ptvsd\_vendored\pydevd\pydevd.py", line 1629, in main
    globals = debugger.run(setup['file'], None, None, is_module)
  File "C:\GIT\ptvsd\ptvsd\_vendored\pydevd\pydevd.py", line 1036, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
IOError: [Errno 2] No such file or directory: 'c:\\GIT\\pyscratch\\??a2.py'
```

The error that was seen by @DonJayamanne  is this one:
```
Traceback (most recent call last):
 File "/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/ipcjson.py", line 336, in on_request
   target(request, args)
 File "/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/wrapper.py", line 881, in f
   fut = m(self, self.loop, *args, **kwargs)
 File "/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/futures.py", line 140, in g
   it = f(self, *args, **kwargs)
 File "/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/pythonFiles/experimental/ptvsd/ptvsd/wrapper.py", line 1739, in on_setBreakpoints
   expression, hit_condition, is_logpoint)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 78-79: ordinal not in range(128)
```
That's because you can have paths as `bytes` and not as `str`/`unicode` on Linux. IOW you can have different file paths on the same file system using different encodings, so you basically have to be tolerant of paths never being strings and work with them as bytes as necessary (or we have to be very clear that we don't support that use-case).
@karthiknadig 
Forgot to provide a key bit of information, this doesn't work in Python 2.7

Resolving patch:
diff --git a/ptvsd/wrapper.py b/ptvsd/wrapper.py
--- a/ptvsd/wrapper.py
+++ b/ptvsd/wrapper.py
@@ -66,6 +66,16 @@ def is_debugger_internal_thread(thread_name):
     return False
 
 
+try:
+    unicode  # noqa
+
+    def needs_unicode(value):
+        return isinstance(value, unicode)  # noqa
+except Exception:
+    def needs_unicode(value):
+        return False
+
+
 class SafeReprPresentationProvider(pydevd_extapi.StrPresentationProvider):
     """
     Computes string representation of Python values by delegating them
@@ -396,7 +406,7 @@ def make_packet(self, cmd_id, args):
         with self.lock:
             seq = self.seq
             self.seq += 1
-        s = '{}\t{}\t{}\n'.format(cmd_id, seq, args)
+        s = u'{}\t{}\t{}\n'.format(cmd_id, seq, args)
         return seq, s
 
     def pydevd_notify(self, cmd_id, args):
@@ -1509,7 +1519,8 @@ def on_stackTrace(self, request, args):
             key = (pyd_tid, int(xframe['id']))
             fid = self.frame_map.to_vscode(key, autogen=True)
             name = unquote(xframe['name'])
-            norm_path = self.path_casing.un_normcase(unquote(xframe['file']))
+            # pydevd encodes if necessary and then uses urllib.quote.
+            norm_path = self.path_casing.un_normcase(unquote(str(xframe['file'])))  # noqa
             source_reference = self.get_source_reference(norm_path)
             if not self.internals_filter.is_internal_path(norm_path):
                 module = self.modules_mgr.add_or_get_from_path(norm_path)
@@ -1974,6 +1985,8 @@ def on_setBreakpoints(self, request, args):
 
         cmd = pydevd_comm.CMD_SET_BREAK
         msgfmt = '{}\t{}\t{}\t{}\tNone\t{}\t{}\t{}\t{}'
+        if needs_unicode(path):
+            msgfmt = unicode(msgfmt)   # noqa
         for src_bp in src_bps:
             line = src_bp['line']
             vsc_bpid = self.bp_map.add(


Test patch:
diff --git a/tests/highlevel/test_live_pydevd.py b/tests/highlevel/test_live_pydevd.py
--- a/tests/highlevel/test_live_pydevd.py
+++ b/tests/highlevel/test_live_pydevd.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import contextlib
 import os
 import sys
@@ -460,6 +462,11 @@ def test_exception_breakpoints(self):
         self.assertIn('ka-boom', out)
 
 
+@unittest.skip('Needs fixing when running with code coverage')
+class UnicodeBreakpointTests(BreakpointTests):
+    FILENAME = u'汉语a2.py'
+
+
 class LogpointTests(TestBase, unittest.TestCase):
     FILENAME = 'spam.py'
     SOURCE = """