Repository: ipython/ipython

Issue:
%run modules clobber each other
I have used %run to define functions from two different importable modules in the same notebook (I know, I should use import, but %run is really convenient when you're still changing the file). When I %run each, it seems to clobber the other, so when I call the function from that one, it gives me NameErrors (not for the function itself, but for things referenced in the function).

Also, the traceback goes a bit funny - e.g. when I call `violin_plot()` from `violinplot.py`, having run `average.py` more recently, it says that the stack frame is in `...scripts/average.py in violin_plot(data1, classes, data2, **kwargs)`.

I suspect that the problem is in the machinery that caches module namespaces. And I have a feeling that I'm the last one to have delved into that, so I'll try to investigate it at some point.



Discussion:
There are a few too many nuances for me to fully grok, can you include a working example?

Here's a minimal example, I'm just about to look into it.

```
$ cat a.py 
avar = 1

def afunc():
    return avar
$ cat b.py
bvar = 2

def bfunc():
    return bvar
$ ipython
...

In [1]: %run a

In [2]: afunc()
Out[2]: 1

In [3]: %run b

In [4]: bfunc()
Out[4]: 2

In [5]: afunc()
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-5-c4f77d893f1d> in <module>()
----> 1 afunc()

/home/thomas/scratch/a.py in afunc()
      2 
      3 def afunc():
----> 4     return avar

NameError: global name 'avar' is not defined
```

The traceback is correct this time: I'm not sure what's changed there.


Resolving patch:
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -819,53 +819,32 @@ def register_post_execute(self, func):
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
-    def new_main_mod(self,ns=None):
+    def new_main_mod(self, filename):
         """Return a new 'main' module object for user code execution.
-        """
-        main_mod = self._user_main_module
-        init_fakemod_dict(main_mod,ns)
-        return main_mod
-
-    def cache_main_mod(self,ns,fname):
-        """Cache a main module's namespace.
-
-        When scripts are executed via %run, we must keep a reference to the
-        namespace of their __main__ module (a FakeModule instance) around so
-        that Python doesn't clear it, rendering objects defined therein
-        useless.
+        
+        ``filename`` should be the path of the script which will be run in the
+        module. Requests with the same filename will get the same module, with
+        its namespace cleared.
+        
+        When scripts are executed via %run, we must keep a reference to their
+        __main__ module (a FakeModule instance) around so that Python doesn't
+        clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
-        absolute path of the module object (which corresponds to the script
-        path).  This way, for multiple executions of the same script we only
-        keep one copy of the namespace (the last one), thus preventing memory
-        leaks from old references while allowing the objects from the last
-        execution to be accessible.
-
-        Note: we can not allow the actual FakeModule instances to be deleted,
-        because of how Python tears down modules (it hard-sets all their
-        references to None without regard for reference counts).  This method
-        must therefore make a *copy* of the given namespace, to allow the
-        original module's __dict__ to be cleared and reused.
-
-
-        Parameters
-        ----------
-          ns : a namespace (a dict, typically)
-
-          fname : str
-            Filename associated with the namespace.
-
-        Examples
-        --------
-
-        In [10]: import IPython
-
-        In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
-
-        In [12]: IPython.__file__ in _ip._main_ns_cache
-        Out[12]: True
+        absolute path of the script. This way, for multiple executions of the
+        same script we only keep one copy of the namespace (the last one),
+        thus preventing memory leaks from old references while allowing the
+        objects from the last execution to be accessible.
         """
-        self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
+        filename = os.path.abspath(filename)
+        try:
+            main_mod = self._main_mod_cache[filename]
+        except KeyError:
+            main_mod = self._main_mod_cache[filename] = FakeModule()
+        else:
+            init_fakemod_dict(main_mod)
+        
+        return main_mod
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
@@ -877,17 +856,17 @@ def clear_main_mod_cache(self):
 
         In [15]: import IPython
 
-        In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
+        In [16]: m = _ip.new_main_mod(IPython.__file__)
 
-        In [17]: len(_ip._main_ns_cache) > 0
+        In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
-        In [19]: len(_ip._main_ns_cache) == 0
+        In [19]: len(_ip._main_mod_cache) == 0
         Out[19]: True
         """
-        self._main_ns_cache.clear()
+        self._main_mod_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
@@ -1017,10 +996,7 @@ def init_create_namespaces(self, user_module=None, user_ns=None):
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
-        self._main_ns_cache = {}
-        # And this is the single instance of FakeModule whose __dict__ we keep
-        # copying and clearing for reuse on each %run
-        self._user_main_module = FakeModule()
+        self._main_mod_cache = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
@@ -1174,8 +1150,8 @@ def all_ns_refs(self):
         
         Note that this does not include the displayhook, which also caches
         objects from the output."""
-        return [self.user_ns, self.user_global_ns,
-                self._user_main_module.__dict__] + self._main_ns_cache.values()
+        return [self.user_ns, self.user_global_ns] + \
+               [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True):
         """Clear all internal namespaces, and attempt to release references to
@@ -1219,9 +1195,6 @@ def reset(self, new_session=True):
         # execution protection
         self.clear_main_mod_cache()
 
-        # Clear out the namespace from the last %run
-        self.new_main_mod()
-
     def del_var(self, varname, by_name=False):
         """Delete a variable from the various namespaces, so that, as
         far as possible, we're not keeping any hidden references to it.
diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py
--- a/IPython/core/magics/execution.py
+++ b/IPython/core/magics/execution.py
@@ -508,7 +508,7 @@ def run(self, parameter_s='', runner=None,
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
-            main_mod = self.shell.new_main_mod(prog_ns)
+            main_mod = self.shell.user_module
         else:
             # Run in a fresh, empty namespace
             if 'n' in opts:
@@ -516,7 +516,10 @@ def run(self, parameter_s='', runner=None,
             else:
                 name = '__main__'
 
-            main_mod = self.shell.new_main_mod()
+            # The shell MUST hold a reference to prog_ns so after %run
+            # exits, the python deletion mechanism doesn't zero it out
+            # (leaving dangling references). See interactiveshell for details
+            main_mod = self.shell.new_main_mod(filename)
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
 
@@ -593,10 +596,6 @@ def run():
                 if 'i' in opts:
                     self.shell.user_ns['__name__'] = __name__save
                 else:
-                    # The shell MUST hold a reference to prog_ns so after %run
-                    # exits, the python deletion mechanism doesn't zero it out
-                    # (leaving dangling references).
-                    self.shell.cache_main_mod(prog_ns, filename)
                     # update IPython interactive namespace
 
                     # Some forms of read errors on the file may mean the


Test patch:
diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py
--- a/IPython/core/tests/test_run.py
+++ b/IPython/core/tests/test_run.py
@@ -244,7 +244,6 @@ def test_obj_del(self):
             err = None
         tt.ipexec_validate(self.fname, 'object A deleted', err)
     
-    @dec.skip_known_failure 
     def test_aggressive_namespace_cleanup(self):
         """Test that namespace cleanup is not too aggressive GH-238
 
@@ -258,11 +257,26 @@ class secondtmp(tt.TempFileMixin): pass
                "   try:\n"
                "       ip.magic('run %s')\n"
                "   except NameError as e:\n"
-               "       print i;break\n" % empty.fname)
-        self.mktmp(py3compat.doctest_refactor_print(src))
+               "       print(i)\n"
+               "       break\n" % empty.fname)
+        self.mktmp(src)
         _ip.magic('run %s' % self.fname)
         _ip.run_cell('ip == get_ipython()')
-        nt.assert_equal(_ip.user_ns['i'], 5)
+        nt.assert_equal(_ip.user_ns['i'], 4)
+    
+    def test_run_second(self):
+        """Test that running a second file doesn't clobber the first, gh-3547
+        """
+        self.mktmp("avar = 1\n"
+                   "def afunc():\n"
+                   "  return avar\n")
+
+        empty = tt.TempFileMixin()
+        empty.mktmp("")
+        
+        _ip.magic('run %s' % self.fname)
+        _ip.magic('run %s' % empty.fname)
+        nt.assert_equal(_ip.user_ns['afunc'](), 1)
 
     @dec.skip_win32
     def test_tclass(self):