Repository: spyder-ide/spyder

Issue:
_kill_kernel changed in jupyter_clients 7.0.6
## Description

### What steps will reproduce the problem?

<!--- You can use Markdown here --->

close spyder

### Traceback
```python-traceback
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/manager.py", line 380, in _async_finish_shutdown
    await asyncio.wait_for(
  File "/usr/local/Cellar/python@3.8/3.8.6/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/tasks.py", line 498, in wait_for
    raise exceptions.TimeoutError()
asyncio.exceptions.TimeoutError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/utils.py", line 26, in wrapped
    raise e
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/utils.py", line 23, in wrapped
    return loop.run_until_complete(future)
  File "/usr/local/lib/python3.8/site-packages/nest_asyncio.py", line 70, in run_until_complete
    return f.result()
  File "/usr/local/Cellar/python@3.8/3.8.6/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/futures.py", line 178, in result
    raise self._exception
  File "/usr/local/Cellar/python@3.8/3.8.6/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/tasks.py", line 280, in __step
    result = coro.send(None)
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/manager.py", line 443, in _async_shutdown_kernel
    await ensure_async(self.finish_shutdown(restart=restart))
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/utils.py", line 26, in wrapped
    raise e
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/utils.py", line 23, in wrapped
    return loop.run_until_complete(future)
  File "/usr/local/lib/python3.8/site-packages/nest_asyncio.py", line 70, in run_until_complete
    return f.result()
  File "/usr/local/Cellar/python@3.8/3.8.6/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/futures.py", line 178, in result
    raise self._exception
  File "/usr/local/Cellar/python@3.8/3.8.6/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/tasks.py", line 280, in __step
    result = coro.send(None)
  File "/usr/local/lib/python3.8/site-packages/jupyter_client/manager.py", line 386, in _async_finish_shutdown
    await ensure_async(self._kill_kernel(restart=restart))
TypeError: _kill_kernel() got an unexpected keyword argument 'restart'
```

## Versions

* Spyder version: 6.0.0.dev0 a84c40648
* Python version: 3.8.6
* Qt version: 5.12.10
* PyQt5 version: 5.12.3
* Operating System: Darwin 20.6.0

### Dependencies

```

# Mandatory:
applaunchservices >=0.1.7              :  0.2.1 (OK)
atomicwrites >=1.2.0                   :  1.4.0 (OK)
chardet >=2.0.0                        :  3.0.4 (OK)
cloudpickle >=0.5.0                    :  2.0.0 (OK)
cookiecutter >=1.6.0                   :  1.7.2 (OK)
diff_match_patch >=20181111            :  20200713 (OK)
intervaltree >=3.0.2                   :  3.1.0 (OK)
IPython >=7.6.0                        :  7.29.0 (OK)
jedi >=0.17.2;<0.19.0                  :  0.18.0 (OK)
jellyfish >=0.7                        :  None (OK)
jsonschema >=3.2.0                     :  3.2.0 (OK)
keyring >=17.0.0                       :  21.4.0 (OK)
nbconvert >=4.0                        :  6.0.7 (OK)
numpydoc >=0.6.0                       :  1.1.0 (OK)
parso >=0.7.0;<0.9.0                   :  0.8.2 (OK)
pexpect >=4.4.0                        :  4.8.0 (OK)
pickleshare >=0.4                      :  0.7.5 (OK)
psutil >=5.3                           :  5.7.2 (OK)
pygments >=2.0                         :  2.10.0 (OK)
pylint >=2.5.0                         :  2.6.0 (OK)
pyls_spyder >=0.4.0                    :  0.4.0 (OK)
pylsp >=1.2.2;<1.3.0                   :  1.1.0 (NOK)
pylsp_black >=1.0.0                    :  None (OK)
qdarkstyle =3.0.2                      :  3.0.2 (OK)
qstylizer >=0.1.10                     :  0.1.10 (OK)
qtawesome >=1.0.2                      :  1.0.2 (OK)
qtconsole >=5.1.0                      :  5.2.0.dev0 (OK)
qtpy >=1.5.0                           :  1.9.0 (OK)
rtree >=0.9.7                          :  None (OK)
setuptools >=49.6.0                    :  50.3.0 (OK)
sphinx >=0.6.6                         :  3.2.1 (OK)
spyder_kernels >=2.1.1;<2.2.0          :  2.2.0.dev0 (OK)
textdistance >=4.2.0                   :  4.2.0 (OK)
three_merge >=0.1.1                    :  0.1.1 (OK)
watchdog >=0.10.3                      :  0.10.3 (OK)
zmq >=17                               :  22.3.0 (OK)

# Optional:
cython >=0.21                          :  0.29.21 (OK)
matplotlib >=2.0.0                     :  3.4.2 (OK)
numpy >=1.7                            :  1.21.0 (OK)
pandas >=1.1.1                         :  1.1.3 (OK)
scipy >=0.17.0                         :  1.5.2 (OK)
sympy >=0.7.3                          :  1.6.2 (OK)

# Spyder plugins:
spyder_boilerplate.spyder.plugin 0.0.1 :  0.0.1 (OK)
```



Discussion:

Resolving patch:
diff --git a/external-deps/spyder-kernels/setup.py b/external-deps/spyder-kernels/setup.py
--- a/external-deps/spyder-kernels/setup.py
+++ b/external-deps/spyder-kernels/setup.py
@@ -43,7 +43,7 @@ def get_version(module='spyder_kernels'):
     'ipykernel>=5.3.0; python_version>="3"',
     'ipython<6; python_version<"3"',
     'ipython>=7.6.0; python_version>="3"',
-    'jupyter-client>=5.3.4,<7',
+    'jupyter-client>=5.3.4',
     'pyzmq>=17',
     'wurlitzer>=1.0.3;platform_system!="Windows"',
 ]
diff --git a/external-deps/spyder-kernels/spyder_kernels/console/shell.py b/external-deps/spyder-kernels/spyder_kernels/console/shell.py
--- a/external-deps/spyder-kernels/spyder_kernels/console/shell.py
+++ b/external-deps/spyder-kernels/spyder_kernels/console/shell.py
@@ -11,6 +11,7 @@
 """
 
 # Standard library imports
+import bdb
 import sys
 
 # Third-party imports
@@ -25,12 +26,25 @@ def __init__(self, *args, **kwargs):
         self._pdb_obj = None
         super(SpyderShell, self).__init__(*args, **kwargs)
 
+    # ---- Methods overriden by us.
     def ask_exit(self):
         """Engage the exit actions."""
         self.kernel.frontend_comm.close_thread()
         return super(SpyderShell, self).ask_exit()
 
-    # --- For Pdb namespace integration
+    def _showtraceback(self, etype, evalue, stb):
+        """
+        Don't show a traceback when exiting our debugger after entering
+        it through a `breakpoint()` call.
+
+        This is because calling `!exit` after `breakpoint()` raises
+        BdbQuit, which throws a long and useless traceback.
+        """
+        if etype is bdb.BdbQuit:
+            stb = ['']
+        super(SpyderShell, self)._showtraceback(etype, evalue, stb)
+
+    # ---- For Pdb namespace integration
     def get_local_scope(self, stack_depth):
         """Get local scope at given frame depth."""
         frame = sys._getframe(stack_depth + 1)
diff --git a/spyder/plugins/ipythonconsole/utils/manager.py b/spyder/plugins/ipythonconsole/utils/manager.py
--- a/spyder/plugins/ipythonconsole/utils/manager.py
+++ b/spyder/plugins/ipythonconsole/utils/manager.py
@@ -17,6 +17,12 @@
 from qtconsole.manager import QtKernelManager
 import psutil
 
+# qtconsole and spyder-kernels depend on jupyter_client
+if hasattr(QtKernelManager, 'provisioner'):
+    JUPYTER_CLIENT_GE_7 = True
+    from jupyter_client.utils import run_sync
+else:
+    JUPYTER_CLIENT_GE_7 = False
 
 class SpyderKernelManager(QtKernelManager):
     """
@@ -76,12 +82,13 @@ def kill_proc_tree(pid, sig=signal.SIGTERM, include_parent=True,
 
         return (gone, alive)
 
-    def _kill_kernel(self):
+    def _kill_kernel_5(self):
         """
         Kill the running kernel.
 
-        Override private method to be able to correctly close kernel that was
-        started via a batch/bash script for correct conda env activation.
+        Override private method of jupyter_client 5 to be able to correctly
+        close kernel that was started via a batch/bash script for correct conda
+        env activation.
         """
         if self.has_kernel:
 
@@ -114,3 +121,40 @@ def _kill_kernel(self):
             self.kernel = None
         else:
             raise RuntimeError("Cannot kill kernel. No kernel is running!")
+
+    async def _async_kill_kernel(self, restart: bool = False) -> None:
+        """Kill the running kernel.
+
+        Override private method of jupyter_client 7 to be able to correctly
+        close kernel that was started via a batch/bash script for correct conda
+        env activation.
+        """
+        if self.has_kernel:
+            assert self.provisioner is not None
+
+            # This is the additional line that was added to properly
+            # kill the kernel started by Spyder.
+            self.kill_proc_tree(self.provisioner.process.pid)
+
+            await self.provisioner.kill(restart=restart)
+
+            # Wait until the kernel terminates.
+            import asyncio
+            try:
+                await asyncio.wait_for(self._async_wait(), timeout=5.0)
+            except asyncio.TimeoutError:
+                # Wait timed out, just log warning but continue
+                #  - not much more we can do.
+                self.log.warning("Wait for final termination of kernel timed"
+                                 " out - continuing...")
+                pass
+            else:
+                # Process is no longer alive, wait and clear
+                if self.has_kernel:
+                    await self.provisioner.wait()
+
+    # override alias for jupyter_client < 7
+    if JUPYTER_CLIENT_GE_7:
+        _kill_kernel = run_sync(_async_kill_kernel)
+    else:
+        _kill_kernel = _kill_kernel_5


Test patch:
diff --git a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py
--- a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py
+++ b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py
@@ -57,9 +57,9 @@ def setup_kernel(cmd):
     This function was taken from the ipykernel project.
     We plan to remove it when dropping support for python 2.
 
-    Returns
+    Yields
     -------
-    kernel_manager: connected KernelManager instance
+    client: jupyter_client.BlockingKernelClient connected to the kernel
     """
     kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE)
     try:
@@ -423,7 +423,7 @@ def test_cwd_in_sys_path():
     with setup_kernel(cmd) as client:
         msg_id = client.execute("import sys; sys_path = sys.path",
                                 user_expressions={'output':'sys_path'})
-        reply = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        reply = client.get_shell_msg(timeout=TIMEOUT)
 
         # Transform value obtained through user_expressions
         user_expressions = reply['content']['user_expressions']
@@ -447,7 +447,7 @@ def test_multiprocessing(tmpdir):
     with setup_kernel(cmd) as client:
         # Remove all variables
         client.execute("%reset -f")
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Write multiprocessing code to a file
         code = """
@@ -465,11 +465,11 @@ def f(x):
 
         # Run code
         client.execute("runfile(r'{}')".format(to_text_string(p)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Verify that the `result` variable is defined
         client.inspect('result')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
@@ -487,7 +487,7 @@ def test_dask_multiprocessing(tmpdir):
     with setup_kernel(cmd) as client:
         # Remove all variables
         client.execute("%reset -f")
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Write multiprocessing code to a file
         # Runs two times to verify that in the second case it doesn't break
@@ -504,14 +504,14 @@ def test_dask_multiprocessing(tmpdir):
 
         # Run code two times
         client.execute("runfile(r'{}')".format(to_text_string(p)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         client.execute("runfile(r'{}')".format(to_text_string(p)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Verify that the `x` variable is defined
         client.inspect('x')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
@@ -527,7 +527,7 @@ def test_runfile(tmpdir):
     with setup_kernel(cmd) as client:
         # Remove all variables
         client.execute("%reset -f")
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Write defined variable code to a file
         code = u"result = 'hello world'; error # make an error"
@@ -547,40 +547,40 @@ def test_runfile(tmpdir):
         # Run code file `d` to define `result` even after an error
         client.execute("runfile(r'{}', current_namespace=False)"
                        .format(to_text_string(d)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Verify that `result` is defined in the current namespace
         client.inspect('result')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
         # Run code file `u` without current namespace
         client.execute("runfile(r'{}', current_namespace=False)"
                        .format(to_text_string(u)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Verify that the variable `result2` is defined
         client.inspect('result2')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
         # Run code file `u` with current namespace
         client.execute("runfile(r'{}', current_namespace=True)"
                        .format(to_text_string(u)))
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
 
         # Verify that the variable `result3` is defined
         client.inspect('result3')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
         # Verify that the variable `__file__` is undefined
         client.inspect('__file__')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert not content['found']
 
@@ -601,14 +601,14 @@ def test_np_threshold(kernel):
     suppress=True,
     formatter={'float_kind':'{:0.2f}'.format})
     """)
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Create a big Numpy array and an array to check decimal format
         client.execute("""
 x = np.random.rand(75000,5);
 a = np.array([123412341234.123412341234])
 """)
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Assert that NumPy threshold, suppress and formatter
         # are the same as the ones set by the user
@@ -617,29 +617,29 @@ def test_np_threshold(kernel):
 s = np.get_printoptions()['suppress'];
 f = np.get_printoptions()['formatter']
 """)
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Check correct decimal format
         client.inspect('a')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']['data']['text/plain']
         assert "123412341234.12" in content
 
         # Check threshold value
         client.inspect('t')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']['data']['text/plain']
         assert "inf" in content
 
         # Check suppress value
         client.inspect('s')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']['data']['text/plain']
         assert "True" in content
 
         # Check formatter
         client.inspect('f')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']['data']['text/plain']
         assert "{'float_kind': <built-in method format of str object" in content
 
@@ -667,7 +667,7 @@ def test_turtle_launch(tmpdir):
     with setup_kernel(cmd) as client:
         # Remove all variables
         client.execute("%reset -f")
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Write turtle code to a file
         code = """
@@ -692,11 +692,11 @@ def test_turtle_launch(tmpdir):
 
         # Run code
         client.execute("runfile(r'{}')".format(to_text_string(p)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Verify that the `tess` variable is defined
         client.inspect('tess')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
@@ -708,11 +708,11 @@ def test_turtle_launch(tmpdir):
 
         # Run code again
         client.execute("runfile(r'{}')".format(to_text_string(p)))
-        client.get_shell_msg(block=True, timeout=TIMEOUT)
+        client.get_shell_msg(timeout=TIMEOUT)
 
         # Verify that the `a` variable is defined
         client.inspect('a')
-        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
 
@@ -727,7 +727,7 @@ def test_matplotlib_inline(kernel):
         # Get current backend
         code = "import matplotlib; backend = matplotlib.get_backend()"
         client.execute(code, user_expressions={'output': 'backend'})
-        reply = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        reply = client.get_shell_msg(timeout=TIMEOUT)
 
         # Transform value obtained through user_expressions
         user_expressions = reply['content']['user_expressions']
diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py
--- a/spyder/app/tests/test_mainwindow.py
+++ b/spyder/app/tests/test_mainwindow.py
@@ -1011,12 +1011,8 @@ def test_connection_to_external_kernel(main_window, qtbot):
     qtbot.wait(1000)
 
     # Make sure everything quit properly
-    assert km.kernel.poll() is not None
-    assert spykm.kernel.poll() is not None
-    if spykm._restarter:
-        assert spykm._restarter.poll() is not None
-    if km._restarter:
-        assert km._restarter.poll() is not None
+    assert not km.is_alive()
+    assert not spykm.is_alive()
 
     # Close the channels
     spykc.stop_channels()