Repository: spyder-ide/spyder

Issue:
`%runcell` can not edit locals
<!--- **PLEASE READ:** When submitting here, please ensure you've completed the following checklist and checked the boxes to confirm. Issue reports without it may be closed. Thanks! --->

### Issue Report Checklist

* [ ] Searched the [issues page](https://github.com/spyder-ide/spyder/issues?q=is%3Aissue) for similar reports
* [ ] Read the relevant sections of the [Spyder Troubleshooting Guide](https://github.com/spyder-ide/spyder/wiki/Troubleshooting-Guide-and-FAQ) and followed its advice
* [ ] Reproduced the issue after updating with ``conda update spyder`` (or ``pip``, if not using Anaconda)
* [ ] Could not reproduce inside ``jupyter qtconsole`` (if console-related)
* [ ] Tried basic troubleshooting (if a bug/error)
    * [ ] Restarted Spyder
    * [ ] Reset preferences with ``spyder --reset``
    * [ ] Reinstalled the latest version of [Anaconda](https://www.anaconda.com/download/)
    * [ ] Tried the other applicable steps from the Troubleshooting Guide
* [ ] Completed the **Problem Description**, **Steps to Reproduce** and **Version** sections below


## Problem Description



### What steps reproduce the problem?

1. debug this code with a breakpoint on line 3: 
```
def fun():
    rr = 1
    aa = 2
fun()
#%%
rr = 3
print(rr)
``` 
```
IPdb [1]: rr
Out  [1]: 1

IPdb [2]: rr = 2

IPdb [3]: rr
Out  [3]: 2

IPdb [4]: %runcell -i 1
3

IPdb [5]: rr
Out  [5]: 2

IPdb [6]: 
```
runcell failed to modify the value of rr!

### What is the expected output? What do you see instead?



### Paste Traceback/Error Below (if applicable)
<!--- Copy from error dialog or View > Panes > Internal Console --->

```python-traceback

PASTE TRACEBACK HERE

```

## Versions
<!--- You can get this information from Help > About Spyder...
or (if Spyder won't launch) the "conda list" command
from the Anaconda Prompt/Terminal/command line. --->

* Spyder version: 
* Python version: 
* Qt version: 
* PyQt version: 
* Operating System name/version: 

### Dependencies
<!--- Please go to the menu entry Help > Dependencies,
press the Copy to clipboard button and paste below --->

```
PASTE DEPENDENCIES HERE
```



Discussion:

Resolving patch:
diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py b/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py
--- a/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py
+++ b/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py
@@ -41,7 +41,9 @@
 from spyder_kernels.customize.namespace_manager import NamespaceManager
 from spyder_kernels.customize.spyderpdb import SpyderPdb
 from spyder_kernels.customize.umr import UserModuleReloader
-from spyder_kernels.customize.utils import capture_last_Expr, canonic
+from spyder_kernels.customize.utils import (
+    capture_last_Expr, canonic, exec_encapsulate_locals
+)
 
 
 # For logging
@@ -496,11 +498,12 @@ def _exec_code(
 
             if capture_last_expression:
                 ast_code, capture_last_expression = capture_last_Expr(
-                    ast_code, "_spyder_out"
+                    ast_code, "_spyder_out", ns_globals
                 )
-                ns_globals["__spyder_builtins__"] = builtins
 
-            exec_fun(compile(ast_code, filename, "exec"), ns_globals, ns_locals)
+            exec_encapsulate_locals(
+                ast_code, ns_globals, ns_locals, exec_fun, filename
+            )
 
             if capture_last_expression:
                 out = ns_globals.pop("_spyder_out", None)
diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py b/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py
--- a/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py
+++ b/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py
@@ -25,7 +25,9 @@
 
 import spyder_kernels
 from spyder_kernels.comms.frontendcomm import CommError, frontend_request
-from spyder_kernels.customize.utils import path_is_library, capture_last_Expr
+from spyder_kernels.customize.utils import (
+    path_is_library, capture_last_Expr, exec_encapsulate_locals
+)
 
 
 logger = logging.getLogger(__name__)
@@ -142,22 +144,22 @@ def default(self, line):
             if cmd == "debug":
                 return self.do_debug(arg)
 
-        locals = self.curframe_locals
-        globals = self.curframe.f_globals
+        local_ns = self.curframe_locals
+        global_ns = self.curframe.f_globals
 
         if self.pdb_use_exclamation_mark:
             # Find pdb commands executed without !
             cmd, arg, line = self.parseline(line)
             if cmd:
                 cmd_in_namespace = (
-                    cmd in globals
-                    or cmd in locals
+                    cmd in global_ns
+                    or cmd in local_ns
                     or cmd in builtins.__dict__
                 )
                 # Special case for quit and exit
                 if cmd in ("quit", "exit"):
-                    if cmd in globals and isinstance(
-                            globals[cmd], ZMQExitAutocall):
+                    if cmd in global_ns and isinstance(
+                            global_ns[cmd], ZMQExitAutocall):
                         # Use the pdb call
                         cmd_in_namespace = False
                 cmd_func = getattr(self, 'do_' + cmd, None)
@@ -181,6 +183,7 @@ def default(self, line):
                         # The pdb command is masked by something
                         self.print_exclamation_warning()
         try:
+            is_magic = line.startswith("%")
             line = TransformerManager().transform_cell(line)
             save_stdout = sys.stdout
             save_stdin = sys.stdin
@@ -199,82 +202,19 @@ def default(self, line):
                     capture_last_expression = False
                 else:
                     code_ast, capture_last_expression = capture_last_Expr(
-                        code_ast, "_spyderpdb_out")
-
-                globals["__spyder_builtins__"] = builtins
-
-                if locals is not globals:
-                    # Mitigates a behaviour of CPython that makes it difficult
-                    # to work with exec and the local namespace
-                    # See:
-                    #  - https://bugs.python.org/issue41918
-                    #  - https://bugs.python.org/issue46153
-                    #  - https://bugs.python.org/issue21161
-                    #  - spyder-ide/spyder#13909
-                    #  - spyder-ide/spyder-kernels#345
-                    #
-                    # The idea here is that the best way to emulate being in a
-                    # function is to actually execute the code in a function.
-                    # A function called `_spyderpdb_code` is created and
-                    # called. It will first load the locals, execute the code,
-                    # and then update the locals.
-                    #
-                    # One limitation of this approach is that locals() is only
-                    # a copy of the curframe locals. This means that closures
-                    # for example are early binding instead of late binding.
-
-                    # Create a function
-                    indent = "    "
-                    code = ["def _spyderpdb_code():"]
-
-                    # Add locals in globals
-                    # If the debugger is recursive, the globals could already
-                    # have a _spyderpdb_locals as it might be shared between
-                    # levels
-                    if "_spyderpdb_locals" in globals:
-                        globals["_spyderpdb_locals"].append(locals)
-                    else:
-                        globals["_spyderpdb_locals"] = [locals]
-
-                    # Load locals if they have a valid name
-                    # In comprehensions, locals could contain ".0" for example
-                    code += [indent + "{k} = _spyderpdb_locals[-1]['{k}']".format(
-                        k=k) for k in locals if k.isidentifier()]
-
-                    # The code comes here
-
-                    # Update the locals
-                    code += [indent + "_spyderpdb_locals[-1].update("
-                             "__spyder_builtins__.locals())"]
-
-                    # Run the function
-                    code += ["_spyderpdb_code()"]
-
-                    # Parse the function
-                    fun_ast = ast.parse('\n'.join(code) + '\n')
-
-                    # Inject code_ast in the function before the locals update
-                    fun_ast.body[0].body = (
-                        fun_ast.body[0].body[:-1]  # The locals
-                        + code_ast.body  # Code to run
-                        + fun_ast.body[0].body[-1:]  # Locals update
-                    )
-                    code_ast = fun_ast
-
-                try:
-                    exec(compile(code_ast, "<stdin>", "exec"), globals)
-                finally:
-                    if locals is not globals:
-                        # CLeanup code
-                        globals.pop("_spyderpdb_code", None)
-                        if len(globals["_spyderpdb_locals"]) > 1:
-                            del globals["_spyderpdb_locals"][-1]
-                        else:
-                            del globals["_spyderpdb_locals"]
-                        
+                        code_ast, "_spyderpdb_out", global_ns)
+
+                if is_magic:
+                    # Magics like runcell use and modify local_ns.
+                    # But the locals() dict can not be directly modified when
+                    # encapsulated. Therefore they must encapsulate the locals
+                    # themselves (see code_runner.py).
+                    exec(compile(code_ast, "<stdin>", "exec"), global_ns, local_ns)
+                else:
+                    exec_encapsulate_locals(code_ast, global_ns, local_ns)
 
                 if capture_last_expression:
-                    out = globals.pop("_spyderpdb_out", None)
+                    out = global_ns.pop("_spyderpdb_out", None)
                     if out is not None:
                         sys.stdout.flush()
                         sys.stderr.flush()
@@ -360,7 +300,7 @@ def stop_here(self, frame):
             # This is spyder-kernels internals
             return False
         return True
-    
+
     def should_continue(self, frame):
         """
         Jump to first breakpoint if needed.
@@ -617,9 +557,9 @@ def do_debug(self, arg):
         with self.recursive_debugger() as debugger:
             self.message("Entering recursive debugger")
             try:
-                globals = self.curframe.f_globals
-                locals = self.curframe_locals
-                return sys.call_tracing(debugger.run, (arg, globals, locals))
+                global_ns = self.curframe.f_globals
+                local_ns = self.curframe_locals
+                return sys.call_tracing(debugger.run, (arg, global_ns, local_ns))
             except Exception:
                 exc_info = sys.exc_info()[:2]
                 self.error(
diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/utils.py b/external-deps/spyder-kernels/spyder_kernels/customize/utils.py
--- a/external-deps/spyder-kernels/spyder_kernels/customize/utils.py
+++ b/external-deps/spyder-kernels/spyder_kernels/customize/utils.py
@@ -6,6 +6,7 @@
 """Utility functions."""
 
 import ast
+import builtins
 import os
 import re
 import sys
@@ -96,18 +97,15 @@ def path_is_library(path, initial_pathlist=None):
         return False
 
 
-def capture_last_Expr(code_ast, out_varname):
-    """
-    Parse line and modify code to capture in globals the last expression.
-
-    The namespace must contain __spyder_builtins__, which is the builtins module.
-    """
+def capture_last_Expr(code_ast, out_varname, global_ns):
+    """Parse line and modify code to capture in globals the last expression."""
     # Modify ast code to capture the last expression
     capture_last_expression = False
     if (
         len(code_ast.body)
         and isinstance(code_ast.body[-1], ast.Expr)
     ):
+        global_ns["__spyder_builtins__"] = builtins
         capture_last_expression = True
         expr_node = code_ast.body[-1]
         # Create new assign node
@@ -131,6 +129,94 @@ def capture_last_Expr(code_ast, out_varname):
     return code_ast, capture_last_expression
 
 
+def exec_encapsulate_locals(
+    code_ast, globals, locals, exec_fun=None, filename=None
+):
+    """
+    Execute by encapsulating locals if needed.
+
+    Notes
+    ----- 
+    * In general, the dict returned by locals() might or might not be modified.
+      In this case, the encapsulated dict can not.
+    """
+    use_locals_hack = locals is not None and locals is not globals
+    if use_locals_hack:
+        globals["__spyder_builtins__"] = builtins
+
+        # Mitigates a behaviour of CPython that makes it difficult
+        # to work with exec and the local namespace
+        # See:
+        #  - https://bugs.python.org/issue41918
+        #  - https://bugs.python.org/issue46153
+        #  - https://bugs.python.org/issue21161
+        #  - spyder-ide/spyder#13909
+        #  - spyder-ide/spyder-kernels#345
+        #
+        # The idea here is that the best way to emulate being in a
+        # function is to actually execute the code in a function.
+        # A function called `_spyderpdb_code` is created and
+        # called. It will first load the locals, execute the code,
+        # and then update the locals.
+        #
+        # One limitation of this approach is that locals() is only
+        # a copy of the curframe locals. This means that closures
+        # for example are early binding instead of late binding.
+
+        # Create a function
+        indent = "    "
+        code = ["def _spyderpdb_code():"]
+
+        # Add locals in globals
+        # If the debugger is recursive, the globals could already
+        # have a _spyderpdb_locals as it might be shared between
+        # levels
+        if "_spyderpdb_locals" in globals:
+            globals["_spyderpdb_locals"].append(locals)
+        else:
+            globals["_spyderpdb_locals"] = [locals]
+
+        # Load locals if they have a valid name
+        # In comprehensions, locals could contain ".0" for example
+        code += [indent + "{k} = _spyderpdb_locals[-1]['{k}']".format(
+            k=k) for k in locals if k.isidentifier()]
+
+        # The code comes here
+
+        # Update the locals
+        code += [indent + "_spyderpdb_locals[-1].update("
+                 "__spyder_builtins__.locals())"]
+
+        # Run the function
+        code += ["_spyderpdb_code()"]
+
+        # Parse the function
+        fun_ast = ast.parse('\n'.join(code) + '\n')
+
+        # Inject code_ast in the function before the locals update
+        fun_ast.body[0].body = (
+            fun_ast.body[0].body[:-1]  # The locals
+            + code_ast.body  # Code to run
+            + fun_ast.body[0].body[-1:]  # Locals update
+        )
+        code_ast = fun_ast
+
+    try:
+        if exec_fun is None:
+            exec_fun = exec
+        if filename is None:
+            filename = "<stdin>"
+        exec_fun(compile(code_ast, filename, "exec"), globals)
+    finally:
+        if use_locals_hack:
+            # Cleanup code
+            globals.pop("_spyderpdb_code", None)
+            if len(globals["_spyderpdb_locals"]) > 1:
+                del globals["_spyderpdb_locals"][-1]
+            else:
+                del globals["_spyderpdb_locals"]
+
+
 def canonic(filename):
     """
     Return canonical form of filename.


Test patch:
diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py
--- a/spyder/app/tests/test_mainwindow.py
+++ b/spyder/app/tests/test_mainwindow.py
@@ -4538,6 +4538,9 @@ def hello():
     test = 1
     print('test ==', test)
 hello()
+#%%
+test = 9
+print([test for i in range(3)])
 """)
 
     # Wait until the window is fully up
@@ -4599,6 +4602,25 @@ def hello():
     assert ('test' in nsb.editor.source_model._data and
             nsb.editor.source_model._data['test']['view'] == '3')
 
+    # Run magic
+    with qtbot.waitSignal(shell.executed):
+        shell.execute("%runcell -i 1")
+
+    qtbot.waitUntil(lambda: "[9, 9, 9]" in shell._control.toPlainText(),
+                    timeout=SHELL_TIMEOUT)
+
+    # check value of test
+    with qtbot.waitSignal(shell.executed):
+        shell.execute("print('test =', test)")
+
+    qtbot.waitUntil(lambda: "test = 9" in shell._control.toPlainText(),
+                    timeout=SHELL_TIMEOUT)
+    assert "test = 9" in shell._control.toPlainText()
+
+    # Check the namespace browser is updated
+    assert ('test' in nsb.editor.source_model._data and
+            nsb.editor.source_model._data['test']['view'] == '9')
+
 
 @flaky(max_runs=3)
 @pytest.mark.use_introspection
@@ -6669,33 +6691,33 @@ def test_quotes_rename_ipy(main_window, qtbot, tmpdir):
     assert "801" in control.toPlainText()
     assert "error" not in control.toPlainText()
     assert "\\.ipy" in control.toPlainText()
-    
+
     # Create an untitled file
     main_window.editor.new()
-    
+
     assert "untitled" in main_window.editor.get_current_filename()
-    
+
     code_editor = main_window.editor.get_focus_widget()
     code_editor.set_text("print(20 + 780)")
-    
+
     with qtbot.waitSignal(shell.executed):
         qtbot.mouseClick(main_window.run_cell_button, Qt.LeftButton)
 
     assert "800" in control.toPlainText()
     assert "error" not in control.toPlainText()
     assert "untitled" in control.toPlainText()
-    
+
     # Save file in a new folder
     code_editor.set_text("print(19 + 780)")
-    
+
     with tempfile.TemporaryDirectory() as td:
-    
+
         editorstack = main_window.editor.get_current_editorstack()
         editorstack.select_savename = lambda fn: os.path.join(td, "fn.ipy")
         main_window.editor.save()
         with qtbot.waitSignal(shell.executed):
             qtbot.mouseClick(main_window.run_cell_button, Qt.LeftButton)
-    
+
         assert "799" in control.toPlainText()
         assert "error" not in control.toPlainText()
         assert "fn.ipy" in control.toPlainText()