Repository: microsoft/ptvsd

Issue:
Pydevd API to send custom events over DAP



Discussion:
What's the custom event to be sent? (i.e.: the dap protocol in pydevd is very specific to the ptvsd needs, so, I'm not sure it's worth it to send something from ptvsd instead of just sending it from pydevd)
It's #1859 for now, but there will be more - e.g. we wanted to send server logs to the adapter over DAP, and that will also need something.

In general, there's a class of issues where it's the code inside ptvsd rather than the code inside pydevd that needs to notify the adapter and/or IDE. So if pydevd is the one sending events, it'd need to expose a corresponding specialized API hook for every one of those for ptvsd to call when it needs to. It seems like it'd be better to just have a single hook for "here's event body as JSON".

Another reason why that would be preferable is that at some point we might start having other packages building on ptvsd also sending custom events that are specific to their scenarios. Some contexts where this was previously discussed are Jupyter and IoT.
Also warnings (justMyCode==false) to be sent via some custom event ( see #1861).

There is another category, basically send a request to IDE and debugger waits for response (reverse-request). That could be used to dynamically change the setting like JMC. The only limitation right now is that VSCode does not support custom reverse-requests.
I think that #1861 could actually be always sent by pydevd -- the telemetry info (#1859) regarding binaries could probably be sent by pydevd too (without any special api), but it does make sense to send the server logs created from ptvsd from ptvsd itself...

As a note, I'm currently working on this issue.
Python extension plans on using DebugAdapterTracker API for logging ( https://github.com/microsoft/vscode-python/issues/8106 ). Having server logs sent as events is useful to capture all the logs in a single location. This is also helpful in remote scenarios to get detailed pydevd logs.

Resolving patch:
diff --git a/src/ptvsd/_vendored/pydevd/pydevd.py b/src/ptvsd/_vendored/pydevd/pydevd.py
--- a/src/ptvsd/_vendored/pydevd/pydevd.py
+++ b/src/ptvsd/_vendored/pydevd/pydevd.py
@@ -54,7 +54,7 @@
     frame_eval_func, dummy_trace_dispatch)
 import pydev_ipython  # @UnusedImport
 from _pydevd_bundle.pydevd_source_mapping import SourceMapping
-from pydevd_concurrency_analyser.pydevd_concurrency_logger import ThreadingLogger, AsyncioLogger, send_message, cur_time
+from pydevd_concurrency_analyser.pydevd_concurrency_logger import ThreadingLogger, AsyncioLogger, send_concurrency_message, cur_time
 from pydevd_concurrency_analyser.pydevd_thread_wrappers import wrap_threads
 from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame, NORM_PATHS_AND_BASE_CONTAINER, get_abs_path_real_path_and_base_from_file
 from pydevd_file_utils import get_fullname, rPath, get_package_dir
@@ -69,6 +69,7 @@
 
 from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
 from _pydevd_bundle.pydevd_process_net_command import process_net_command
+from _pydevd_bundle.pydevd_net_command import NetCommand
 
 from _pydevd_bundle.pydevd_breakpoints import stop_on_unhandled_exception
 from _pydevd_bundle.pydevd_collect_try_except_info import collect_try_except_info
@@ -2126,11 +2127,11 @@ def run(self, file, globals=None, locals=None, is_module=False, set_trace=True):
         if self.thread_analyser is not None:
             wrap_threads()
             self.thread_analyser.set_start_time(cur_time())
-            send_message("threading_event", 0, t.getName(), thread_id, "thread", "start", file, 1, None, parent=thread_id)
+            send_concurrency_message("threading_event", 0, t.getName(), thread_id, "thread", "start", file, 1, None, parent=thread_id)
 
         if self.asyncio_analyser is not None:
             # we don't have main thread in asyncio graph, so we should add a fake event
-            send_message("asyncio_event", 0, "Task", "Task", "thread", "stop", file, 1, frame=None, parent=None)
+            send_concurrency_message("asyncio_event", 0, "Task", "Task", "thread", "stop", file, 1, frame=None, parent=None)
 
         try:
             if INTERACTIVE_MODE_AVAILABLE:
@@ -2236,6 +2237,29 @@ def add_dap_messages_listener(dap_messages_listener):
     py_db.add_dap_messages_listener(dap_messages_listener)
 
 
+def send_json_message(msg):
+    '''
+    API to send some custom json message.
+
+    :param dict|pydevd_schema.BaseSchema msg:
+        The custom message to be sent.
+
+    :return bool:
+        True if the message was added to the queue to be sent and False otherwise.
+    '''
+    py_db = get_global_debugger()
+    if py_db is None:
+        return False
+
+    writer = py_db.writer
+    if writer is None:
+        return False
+
+    cmd = NetCommand(-1, 0, msg, is_json=True)
+    writer.add_command(cmd)
+    return True
+
+
 def set_debug(setup):
     setup['DEBUG_RECORD_SOCKET_READS'] = True
     setup['DEBUG_TRACE_BREAKPOINTS'] = 1
diff --git a/src/ptvsd/_vendored/pydevd/pydevd_concurrency_analyser/pydevd_concurrency_logger.py b/src/ptvsd/_vendored/pydevd/pydevd_concurrency_analyser/pydevd_concurrency_logger.py
--- a/src/ptvsd/_vendored/pydevd/pydevd_concurrency_analyser/pydevd_concurrency_logger.py
+++ b/src/ptvsd/_vendored/pydevd/pydevd_concurrency_analyser/pydevd_concurrency_logger.py
@@ -79,7 +79,7 @@ def get_text_list_for_frame(frame):
     return cmdTextList
 
 
-def send_message(event_class, time, name, thread_id, type, event, file, line, frame, lock_id=0, parent=None):
+def send_concurrency_message(event_class, time, name, thread_id, type, event, file, line, frame, lock_id=0, parent=None):
     dbg = GlobalDebuggerHolder.global_dbg
     if dbg is None:
         return
@@ -109,7 +109,7 @@ def send_message(event_class, time, name, thread_id, type, event, file, line, fr
 
 def log_new_thread(global_debugger, t):
     event_time = cur_time() - global_debugger.thread_analyser.start_time
-    send_message("threading_event", event_time, t.getName(), get_thread_id(t), "thread",
+    send_concurrency_message("threading_event", event_time, t.getName(), get_thread_id(t), "thread",
              "start", "code_name", 0, None, parent=get_thread_id(t))
 
 
@@ -171,7 +171,7 @@ def log_event(self, frame):
 
                         if real_method == "start":
                             parent = get_thread_id(t)
-                        send_message("threading_event", event_time, name, thread_id, "thread",
+                        send_concurrency_message("threading_event", event_time, name, thread_id, "thread",
                         real_method, back.f_code.co_filename, back.f_lineno, back, parent=parent)
                         # print(event_time, self_obj.getName(), thread_id, "thread",
                         #       real_method, back.f_code.co_filename, back.f_lineno)
@@ -191,7 +191,7 @@ def log_event(self, frame):
                                     send_massage = False
                                     # we can't detect stop after join in Python 2 yet
                                 if send_massage:
-                                    send_message("threading_event", event_time, "Thread", my_thread_id, "thread",
+                                    send_concurrency_message("threading_event", event_time, "Thread", my_thread_id, "thread",
                                                  "stop", my_back.f_code.co_filename, my_back.f_lineno, my_back, parent=None)
 
                 if self_obj.__class__ == ObjectWrapper:
@@ -204,7 +204,7 @@ def log_event(self, frame):
                         # back_back_base is the file, where the method was called froms
                         return
                     if method_name == "__init__":
-                        send_message("threading_event", event_time, t.getName(), get_thread_id(t), "lock",
+                        send_concurrency_message("threading_event", event_time, t.getName(), get_thread_id(t), "lock",
                                      method_name, back.f_code.co_filename, back.f_lineno, back, lock_id=str(id(frame.f_locals["self"])))
                     if "attr" in frame.f_locals and \
                             (frame.f_locals["attr"] in LOCK_METHODS or
@@ -219,12 +219,12 @@ def log_event(self, frame):
                         if real_method == "release_end":
                             # do not log release end. Maybe use it later
                             return
-                        send_message("threading_event", event_time, t.getName(), get_thread_id(t), "lock",
+                        send_concurrency_message("threading_event", event_time, t.getName(), get_thread_id(t), "lock",
                         real_method, back.f_code.co_filename, back.f_lineno, back, lock_id=str(id(self_obj)))
 
                         if real_method in ("put_end", "get_end"):
                             # fake release for queue, cause we don't call it directly
-                            send_message("threading_event", event_time, t.getName(), get_thread_id(t), "lock",
+                            send_concurrency_message("threading_event", event_time, t.getName(), get_thread_id(t), "lock",
                                          "release", back.f_code.co_filename, back.f_lineno, back, lock_id=str(id(self_obj)))
                         # print(event_time, t.getName(), get_thread_id(t), "lock",
                         #       real_method, back.f_code.co_filename, back.f_lineno)
@@ -284,14 +284,14 @@ def log_event(self, frame):
                 if method_name == "set_result":
                     task_id = id(self_obj)
                     task_name = self.task_mgr.get(str(task_id))
-                    send_message("asyncio_event", event_time, task_name, task_name, "thread", "stop", frame.f_code.co_filename,
+                    send_concurrency_message("asyncio_event", event_time, task_name, task_name, "thread", "stop", frame.f_code.co_filename,
                                  frame.f_lineno, frame)
 
                 method_name = back.f_code.co_name
                 if method_name == "__init__":
                     task_id = id(self_obj)
                     task_name = self.task_mgr.get(str(task_id))
-                    send_message("asyncio_event", event_time, task_name, task_name, "thread", "start", frame.f_code.co_filename,
+                    send_concurrency_message("asyncio_event", event_time, task_name, task_name, "thread", "start", frame.f_code.co_filename,
                                  frame.f_lineno, frame)
 
             method_name = frame.f_code.co_name
@@ -302,7 +302,7 @@ def log_event(self, frame):
 
                     if method_name == "acquire":
                         if not self_obj._waiters and not self_obj.locked():
-                            send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                            send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                          method_name + "_begin", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
                         if self_obj.locked():
                             method_name += "_begin"
@@ -311,7 +311,7 @@ def log_event(self, frame):
                     elif method_name == "release":
                         method_name += "_end"
 
-                    send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                    send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                  method_name, frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
 
             if isinstance(self_obj, asyncio.Queue):
@@ -320,20 +320,20 @@ def log_event(self, frame):
                     task_name = self.task_mgr.get(str(task_id))
 
                     if method_name == "put":
-                        send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                        send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                      "acquire_begin", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
                     elif method_name == "_put":
-                        send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                        send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                      "acquire_end", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
-                        send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                        send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                      "release", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
                     elif method_name == "get":
                         back = frame.f_back
                         if back.f_code.co_name != "send":
-                            send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                            send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                          "acquire_begin", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
                         else:
-                            send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                            send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                          "acquire_end", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))
-                            send_message("asyncio_event", event_time, task_name, task_name, "lock",
+                            send_concurrency_message("asyncio_event", event_time, task_name, task_name, "lock",
                                          "release", frame.f_code.co_filename, frame.f_lineno, frame, lock_id=str(id(self_obj)))


Test patch:
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_custom_message.py b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_custom_message.py
new file mode 100644
--- /dev/null
+++ b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_custom_message.py
@@ -0,0 +1,14 @@
+import pydevd
+from _pydevd_bundle._debug_adapter import pydevd_schema
+
+body = pydevd_schema.OutputEventBody('some output', 'my_category')
+event = pydevd_schema.OutputEvent(body)
+pydevd.send_json_message(event)
+
+pydevd.send_json_message({
+    "type": "event",
+    "event": "output",
+    "body": {"output": "some output 2", "category": "my_category2"}
+})
+
+print('TEST SUCEEDED')
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
@@ -3190,6 +3190,24 @@ def update_command_line_args(self, args):
         writer.finished_ok = True
 
 
+def test_send_json_message(case_setup):
+
+    with case_setup.test_file('_debugger_case_custom_message.py') as writer:
+        json_facade = JsonFacade(writer)
+
+        json_facade.write_launch()
+
+        json_facade.write_make_initial_run()
+
+        json_facade.wait_for_json_message(
+            OutputEvent, lambda msg: msg.body.category == 'my_category' and msg.body.output == 'some output')
+
+        json_facade.wait_for_json_message(
+            OutputEvent, lambda msg: msg.body.category == 'my_category2' and msg.body.output == 'some output 2')
+
+        writer.finished_ok = True
+
+
 if __name__ == '__main__':
     pytest.main(['-k', 'test_case_skipping_filters', '-s'])