Repository: spyder-ide/spyder

Issue:
Kernel dies silently and doesn't restart properly.
<!--- **PLEASE READ:** When submitting here, please ensure you've completed the following checklist and checked the boxes to confirm. Issue reports without it may be closed. Thanks! --->

### Issue Report Checklist

* [ ] Searched the [issues page](https://github.com/spyder-ide/spyder/issues?q=is%3Aissue) for similar reports
* [ ] Read the relevant sections of the [Spyder Troubleshooting Guide](https://github.com/spyder-ide/spyder/wiki/Troubleshooting-Guide-and-FAQ) and followed its advice
* [ ] Reproduced the issue after updating with ``conda update spyder`` (or ``pip``, if not using Anaconda)
* [ ] Could not reproduce inside ``jupyter qtconsole`` (if console-related)
* [ ] Tried basic troubleshooting (if a bug/error)
    * [ ] Restarted Spyder
    * [ ] Reset preferences with ``spyder --reset``
    * [ ] Reinstalled the latest version of [Anaconda](https://www.anaconda.com/download/)
    * [ ] Tried the other applicable steps from the Troubleshooting Guide
* [ ] Completed the **Problem Description**, **Steps to Reproduce** and **Version** sections below


## Problem Description



### What steps reproduce the problem?

1. Type in the kernel:
```
import numpy as np
a = np.zeros((2, 18030, 162542200), float) * np.nan
```
2. Wait for python to crash
3. When you get a prompt back, try running a cell.

### What is the expected output? What do you see instead?

spyder:
```
Python 3.7.5 (default, Nov  1 2019, 02:16:23) 
Type "copyright", "credits" or "license" for more information.

IPython 7.10.2 -- An enhanced Interactive Python.

In [1]: import numpy as np
   ...: a = np.zeros((2, 18030, 162542200), float) * np.nan

In [1]: 
```
qtconsole:
```
Jupyter QtConsole 4.7.0.dev0
Python 3.7.5 (default, Nov  1 2019, 02:16:23) 
Type 'copyright', 'credits' or 'license' for more information
IPython 7.10.2 -- An enhanced Interactive Python. Type '?' for help.

In [1]: import numpy as np
   ...: a = np.zeros((2, 18030, 162542200), float) * np.nan

Kernel died, restarting


 
Kernel died, restarting


 
Kernel died, restarting


 

In [1]: 

In [1]: 
```

python3:
```
Python 3.7.5 (default, Nov  1 2019, 02:16:23) 
[Clang 11.0.0 (clang-1100.0.33.8)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import numpy as np
>>> a = np.zeros((2, 18030, 162542200), float) * np.nan
zsh: killed     python3
```


After the prompt returns and I run a cell, I get:
```
In [1]: runcell(0, 'test.py')
This command failed to be executed because an error occurred while trying to get the cell code from Spyder's editor. The error was:


An exception has occurred, use %tb to see the full traceback.

CommError: Can't make a request to a closed comm
```
### Paste Traceback/Error Below (if applicable)
<!--- Copy from error dialog or View > Panes > Internal Console --->

```python-traceback

PASTE TRACEBACK HERE

```

## Versions
<!--- You can get this information from Help > About Spyder...
or (if Spyder won't launch) the "conda list" command
from the Anaconda Prompt/Terminal/command line. --->

* Spyder version: 
* Python version: 
* Qt version: 
* PyQt version: 
* Operating System name/version: 

### Dependencies
<!--- Please go to the menu entry Help > Dependencies,
press the Copy to clipboard button and paste below --->

```
PASTE DEPENDENCIES HERE
```



Discussion:

Resolving patch:
diff --git a/spyder/plugins/ipythonconsole/comms/kernelcomm.py b/spyder/plugins/ipythonconsole/comms/kernelcomm.py
--- a/spyder/plugins/ipythonconsole/comms/kernelcomm.py
+++ b/spyder/plugins/ipythonconsole/comms/kernelcomm.py
@@ -32,6 +32,7 @@ class KernelComm(CommBase, QObject):
     """
 
     _sig_got_reply = Signal()
+    _sig_channel_open = Signal()
     sig_exception_occurred = Signal(str, bool)
 
     def __init__(self):
@@ -43,16 +44,18 @@ def __init__(self):
         self.register_call_handler('_async_error', self._async_error)
         self.register_call_handler('_set_comm_port', self._set_comm_port)
 
-
     def _set_comm_port(self, port):
         """Set comm port."""
         client = self.kernel_client
-        client.comm_port = port
-        identity = client.session.bsession
-        socket = client._create_connected_socket(
-            'comm', identity=identity)
-        client.comm_channel = client.shell_channel_class(
-            socket, client.session, client.ioloop)
+        if not (hasattr(client, 'comm_port') and client.comm_port == port):
+            client.comm_port = port
+            identity = client.session.bsession
+            socket = client._create_connected_socket(
+                'comm', identity=identity)
+            client.comm_channel = client.shell_channel_class(
+                socket, client.session, client.ioloop)
+        # We emit in case we are waiting on this
+        self._sig_channel_open.emit()
 
     def shutdown_comm_channel(self):
         """Shutdown the comm channel."""
@@ -62,12 +65,27 @@ def shutdown_comm_channel(self):
             channel.send(msg)
             self.kernel_client.comm_channel = None
 
+    def comm_channel_connected(self):
+        """Check if the comm channel is connected."""
+        return self.kernel_client.comm_channel is not None
+
     @contextmanager
-    def comm_channel_manager(self, comm_id):
+    def comm_channel_manager(self, comm_id, queue_message=False):
         """Use comm_channel instead of shell_channel."""
-        if not self.kernel_client.comm_channel:
+        if queue_message:
+            # Send without comm_channel
             yield
             return
+
+        if not self.comm_channel_connected():
+            # Ask again for comm config
+            self.remote_call()._send_comm_config()
+            timeout = 10
+            self._wait(self.comm_channel_connected,
+                       self._sig_channel_open,
+                       "Timeout while waiting for comm port.",
+                       timeout)
+
         id_list = self.get_comm_id_list(comm_id)
         for comm_id in id_list:
             self._comms[comm_id]['comm']._send_channel = (
@@ -108,40 +126,51 @@ def _get_call_return_value(self, call_dict, call_data, comm_id):
         """
         settings = call_dict['settings']
         interrupt = 'interrupt' in settings and settings['interrupt']
-        if interrupt:
-            with self.comm_channel_manager(comm_id):
-                return super(KernelComm, self)._get_call_return_value(
-                    call_dict, call_data, comm_id)
-        else:
+
+        with self.comm_channel_manager(comm_id, queue_message=not interrupt):
             return super(KernelComm, self)._get_call_return_value(
                 call_dict, call_data, comm_id)
 
     def _wait_reply(self, call_id, call_name, timeout):
         """Wait for the other side reply."""
-        if call_id in self._reply_inbox:
+
+        def got_reply():
+            return call_id in self._reply_inbox
+
+        timeout_msg = "Timeout while waiting for {}".format(
+            self._reply_waitlist)
+        self._wait(got_reply, self._sig_got_reply, timeout_msg, timeout)
+
+    def _wait(self, condition, signal, timeout_msg, timeout):
+        """
+        Wait until condition() is True by running an event loop.
+
+        signal: qt signal that should interrupt the event loop.
+        timeout_msg: Meaage to display in case of a timeout.
+        timeout: time in seconds before a timeout
+        """
+        if condition():
             return
 
         # Create event loop to wait with
         wait_loop = QEventLoop()
-        self._sig_got_reply.connect(wait_loop.quit)
+        signal.connect(wait_loop.quit)
         wait_timeout = QTimer()
         wait_timeout.setSingleShot(True)
         wait_timeout.timeout.connect(wait_loop.quit)
 
         # Wait until the kernel returns the value
         wait_timeout.start(timeout * 1000)
-        while len(self._reply_waitlist) > 0:
+        while not condition():
             if not wait_timeout.isActive():
-                self._sig_got_reply.disconnect(wait_loop.quit)
-                if call_id in self._reply_waitlist:
-                    raise TimeoutError(
-                        "Timeout while waiting for {}".format(
-                            self._reply_waitlist))
+                signal.disconnect(wait_loop.quit)
+                if not condition():
+                    raise TimeoutError(timeout_msg)
                 return
             wait_loop.exec_()
 
         wait_timeout.stop()
-        self._sig_got_reply.disconnect(wait_loop.quit)
+        signal.disconnect(wait_loop.quit)
 
     def _handle_remote_call_reply(self, msg_dict, buffer, load_exception):
         """
diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py
--- a/spyder/plugins/ipythonconsole/widgets/client.py
+++ b/spyder/plugins/ipythonconsole/widgets/client.py
@@ -123,6 +123,7 @@ def __init__(self, plugin, id_,
         self.allow_rename = True
         self.stderr_dir = None
         self.is_error_shown = False
+        self.restart_thread = None
 
         if css_path is None:
             self.css_path = CSS_PATH
@@ -143,7 +144,7 @@ def __init__(self, plugin, id_,
         # To keep a reference to the page to be displayed
         # in infowidget
         self.info_page = None
-        self._show_loading_page()
+        self._before_prompt_is_ready()
 
         # Elapsed time
         self.time_label = None
@@ -227,7 +228,7 @@ def remove_stderr_file(self):
             pass
 
     def configure_shellwidget(self, give_focus=True):
-        """Configure shellwidget after kernel is started"""
+        """Configure shellwidget after kernel is connected."""
         if give_focus:
             self.get_control().setFocus()
 
@@ -255,8 +256,10 @@ def configure_shellwidget(self, give_focus=True):
         self.shellwidget.executed.connect(self.disable_stop_button)
 
         # To show kernel restarted/died messages
-        self.shellwidget.sig_kernel_restarted.connect(
+        self.shellwidget.sig_kernel_restarted_message.connect(
             self.kernel_restarted_message)
+        self.shellwidget.sig_kernel_restarted.connect(
+            self._finalise_restart)
 
         # To correctly change Matplotlib backend interactively
         self.shellwidget.executing.connect(
@@ -272,12 +275,22 @@ def configure_shellwidget(self, give_focus=True):
         # To apply style
         self.set_color_scheme(self.shellwidget.syntax_style, reset=False)
 
+    def _before_prompt_is_ready(self):
+        """Configure shellwidget before kernel is connected."""
+        self._show_loading_page()
+        self.shellwidget.sig_prompt_ready.connect(
+            self._when_prompt_is_ready)
+
+    def _when_prompt_is_ready(self):
+        """Configuration after the prompt is shown."""
         # To hide the loading page
-        self.shellwidget.sig_prompt_ready.connect(self._hide_loading_page)
+        self._hide_loading_page()
 
         # Show possible errors when setting Matplotlib backend
-        self.shellwidget.sig_prompt_ready.connect(
-            self._show_mpl_backend_errors)
+        self._show_mpl_backend_errors()
+
+        self.shellwidget.sig_prompt_ready.disconnect(
+            self._when_prompt_is_ready)
 
     def enable_stop_button(self):
         self.stop_button.setEnabled(True)
@@ -542,32 +555,54 @@ def restart_kernel(self):
             if sw.kernel_manager:
                 if self.infowidget.isVisible():
                     self.infowidget.hide()
-                    sw.show()
-                try:
-                    # Close comm
-                    sw.spyder_kernel_comm.close()
-                    sw.kernel_manager.restart_kernel(
-                        stderr=self.stderr_handle)
-                    # Reopen comm
-                    sw.spyder_kernel_comm.open_comm(sw.kernel_client)
-
-                except RuntimeError as e:
-                    sw._append_plain_text(
-                        _('Error restarting kernel: %s\n') % e,
-                        before_prompt=True
-                    )
-                else:
-                    # For spyder-ide/spyder#6235, IPython was changing the
-                    # setting of %colors on windows by assuming it was using a
-                    # dark background. This corrects it based on the scheme.
-                    self.set_color_scheme(sw.syntax_style)
-                    sw._append_html(_("<br>Restarting kernel...\n<hr><br>"),
-                                    before_prompt=False)
+                self._show_loading_page()
+
+                # Close comm
+                sw.spyder_kernel_comm.close()
+                self.restart_thread = QThread()
+                self.restart_thread.run = self._restart_thread_main
+                self.restart_thread.error = None
+                self.restart_thread.finished.connect(
+                    lambda: self._finalise_restart(True))
+                self.restart_thread.start()
+
             else:
                 sw._append_plain_text(
                     _('Cannot restart a kernel not started by Spyder\n'),
                     before_prompt=True
                 )
+                self._hide_loading_page()
+
+    def _restart_thread_main(self):
+        """Restart the kernel in a thread."""
+        try:
+            self.shellwidget.kernel_manager.restart_kernel(
+                stderr=self.stderr_handle)
+        except RuntimeError as e:
+            self.restart_thread.error = e
+
+    def _finalise_restart(self, reset=False):
+        """Finishes the restarting of the kernel."""
+        sw = self.shellwidget
+        if self.restart_thread and self.restart_thread.error is not None:
+            sw._append_plain_text(
+                _('Error restarting kernel: %s\n') % self.restart_thread.error,
+                before_prompt=True
+            )
+        else:
+            # Reopen comm
+            sw.spyder_kernel_comm.close()
+            sw.spyder_kernel_comm.open_comm(sw.kernel_client)
+
+            # For spyder-ide/spyder#6235, IPython was changing the
+            # setting of %colors on windows by assuming it was using a
+            # dark background. This corrects it based on the scheme.
+            self.set_color_scheme(sw.syntax_style, reset=reset)
+            sw._append_html(_("<br>Restarting kernel...\n<hr><br>"),
+                            before_prompt=True)
+        if reset:
+            self._hide_loading_page()
+        self.restart_thread = None
 
     @Slot(str)
     def kernel_restarted_message(self, msg):
@@ -709,7 +744,6 @@ def _hide_loading_page(self):
         self.shellwidget.show()
         self.info_page = self.blank_page
         self.set_info_page()
-        self.shellwidget.sig_prompt_ready.disconnect(self._hide_loading_page)
 
     def _read_stderr(self):
         """Read the stderr file of the kernel."""
@@ -741,5 +775,3 @@ def _show_mpl_backend_errors(self):
         """
         if not self.external_kernel:
             self.shellwidget.call_kernel().show_mpl_backend_errors()
-        self.shellwidget.sig_prompt_ready.disconnect(
-            self._show_mpl_backend_errors)
diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py
--- a/spyder/plugins/ipythonconsole/widgets/shell.py
+++ b/spyder/plugins/ipythonconsole/widgets/shell.py
@@ -56,7 +56,8 @@ class ShellWidget(NamepaceBrowserWidget, HelpWidget, DebuggingWidget,
     focus_changed = Signal()
     new_client = Signal()
     sig_is_spykernel = Signal(object)
-    sig_kernel_restarted = Signal(str)
+    sig_kernel_restarted_message = Signal(str)
+    sig_kernel_restarted = Signal()
     sig_prompt_ready = Signal()
 
     # For global working directory
@@ -209,14 +210,10 @@ def set_color_scheme(self, color_scheme, reset=True):
         if not dark_color:
             # Needed to change the colors of tracebacks
             self.silent_execute("%colors linux")
-            self.call_kernel(
-                interrupt=True,
-                blocking=False).set_sympy_forecolor(background_color='dark')
+            self.call_kernel().set_sympy_forecolor(background_color='dark')
         else:
             self.silent_execute("%colors lightbg")
-            self.call_kernel(
-                interrupt=True,
-                blocking=False).set_sympy_forecolor(background_color='light')
+            self.call_kernel().set_sympy_forecolor(background_color='light')
 
     def update_syspath(self, path_dict, new_path_dict):
         """Update sys.path contents on kernel."""
@@ -621,7 +618,11 @@ def _banner_default(self):
 
     def _kernel_restarted_message(self, died=True):
         msg = _("Kernel died, restarting") if died else _("Kernel restarting")
-        self.sig_kernel_restarted.emit(msg)
+        self.sig_kernel_restarted_message.emit(msg)
+
+    def _handle_kernel_restarted(self):
+        super(ShellWidget, self)._handle_kernel_restarted()
+        self.sig_kernel_restarted.emit()
 
     def _syntax_style_changed(self):
         """Refresh the highlighting with the current syntax style by class."""


Test patch:
diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py
--- a/spyder/app/tests/test_mainwindow.py
+++ b/spyder/app/tests/test_mainwindow.py
@@ -235,6 +235,7 @@ def remove_test_dir():
 # =============================================================================
 # ---- Tests
 # =============================================================================
+@flaky(max_runs=3)
 @pytest.mark.slow
 @pytest.mark.single_instance
 @pytest.mark.skipif((os.environ.get('CI', None) is None or (PY2
@@ -256,7 +257,7 @@ def test_single_instance_and_edit_magic(main_window, qtbot, tmpdir):
                  "lock_created = lock.lock()".format(spy_dir_str=spy_dir))
 
     # Test single instance
-    with qtbot.waitSignal(shell.executed):
+    with qtbot.waitSignal(shell.executed, timeout=2000):
         shell.execute(lock_code)
     assert not shell.get_value('lock_created')
 
@@ -2954,5 +2955,33 @@ def test_pbd_step(main_window, qtbot, tmpdir):
         str(test_file))
 
 
+@pytest.mark.slow
+@flaky(max_runs=3)
+def test_runcell_after_restart(main_window, qtbot):
+    """Test runcell after a kernel restart."""
+    # Write code to a file
+    code = "print('test_runcell_after_restart')"
+    # Wait until the window is fully up
+    shell = main_window.ipyconsole.get_current_shellwidget()
+    qtbot.waitUntil(lambda: shell._prompt_html is not None,
+                    timeout=SHELL_TIMEOUT)
+    # create new file
+    main_window.editor.new()
+    code_editor = main_window.editor.get_focus_widget()
+    code_editor.set_text(code)
+
+    # Restart Kernel
+    with qtbot.waitSignal(shell.sig_prompt_ready, timeout=10000):
+        shell.ipyclient.restart_kernel()
+
+    # call runcell
+    code_editor.setFocus()
+    qtbot.keyClick(code_editor, Qt.Key_Return, modifier=Qt.ShiftModifier)
+    qtbot.waitUntil(
+        lambda: "test_runcell_after_restart" in shell._control.toPlainText())
+
+    # Make sure no errors are shown
+    assert "error" not in shell._control.toPlainText().lower()
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/spyder/plugins/ipythonconsole/comms/tests/test_comms.py b/spyder/plugins/ipythonconsole/comms/tests/test_comms.py
--- a/spyder/plugins/ipythonconsole/comms/tests/test_comms.py
+++ b/spyder/plugins/ipythonconsole/comms/tests/test_comms.py
@@ -102,8 +102,15 @@ def comms(kernel):
     frontend_comm = FrontendComm(kernel)
     kernel_comm = KernelComm()
 
+    def dummy_set_comm_port(port):
+        """There is no port to set."""
+        pass
+
+    kernel_comm.register_call_handler('_set_comm_port', dummy_set_comm_port)
+
     class DummyKernelClient():
-        comm_channel = None
+        comm_channel = 0
+        shell_channel = 0
 
     kernel_comm.kernel_client = DummyKernelClient()
 
diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
--- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
+++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
@@ -1621,5 +1621,36 @@ def test_conda_env_activation(ipyconsole, qtbot):
         assert expected_output == output
 
 
+@flaky(max_runs=3)
+@pytest.mark.skipif(os.name == 'nt', reason="no SIGTERM on Windows")
+def test_kernel_kill(ipyconsole, qtbot):
+    """
+    Test that the kernel correctly restarts after a kill.
+    """
+    shell = ipyconsole.get_current_shellwidget()
+    qtbot.waitUntil(lambda: shell._prompt_html is not None,
+                    timeout=SHELL_TIMEOUT)
+    # Wait for the restarter to start
+    qtbot.wait(3000)
+    crash_string = 'import os, signal; os.kill(os.getpid(), signal.SIGTERM)'
+    # Check only one comm is open
+    old_open_comms = list(shell.spyder_kernel_comm._comms.keys())
+    assert len(old_open_comms) == 1
+    with qtbot.waitSignal(shell.sig_prompt_ready, timeout=30000):
+        shell.execute(crash_string)
+    assert crash_string in shell._control.toPlainText()
+    assert "Restarting kernel..." in shell._control.toPlainText()
+    # Check a new comm replaced the old one
+    new_open_comms = list(shell.spyder_kernel_comm._comms.keys())
+    assert len(new_open_comms) == 1
+    assert old_open_comms[0] != new_open_comms[0]
+    # Wait until the comm replies
+    qtbot.waitUntil(
+        lambda: shell.spyder_kernel_comm._comms[new_open_comms[0]][
+            'status'] == 'ready')
+    assert shell.spyder_kernel_comm._comms[new_open_comms[0]][
+        'status'] == 'ready'
+
+
 if __name__ == "__main__":
     pytest.main()
diff --git a/spyder/utils/tests/test_vcs.py b/spyder/utils/tests/test_vcs.py
--- a/spyder/utils/tests/test_vcs.py
+++ b/spyder/utils/tests/test_vcs.py
@@ -29,7 +29,7 @@
 @pytest.mark.skipif(os.environ.get('CI', None) is None,
                     reason="Not to be run outside of CIs")
 def test_vcs_tool():
-    if sys.platform.startswith('linux'):
+    if not os.name == 'nt':
         with pytest.raises(ActionToolNotFound):
             run_vcs_tool(osp.dirname(__file__), 'browse')
     else: