Repository: spyder-ide/spyder

Issue:
Builtins can still be shadowed, causing kernel startup to fail, when a project is open
<!--- **PLEASE READ:** When submitting here, please ensure you've completed the following checklist and checked the boxes to confirm. Issue reports without it may be closed. Thanks! --->

## Problem Description

Opened per discussion with @ccordoba12 . In Python 3.7+, the cwd is added to the `PATH` from which modules are imported, such that Python files named the same as standard library ones shadow them, which results in the kernel not being able to start if it requires it. This is a common error for new users and was partially fixed by removing the home directory, but this evidently does not work when a project is active, as Python modules in the project's root directory will still shadow standard library ones.

### What steps reproduce the problem?

1. Open a project
2. Create a file in the project directory with a name shadowing a stdlib module that the kernel uses, e.g. `code.py`
3. Restart the kernel

### What is the expected output? What do you see instead?

Kernel fails to start. Instead, working directory should be at the bottom of the `PATH` search order so that modules with conflicting names don't shadow builtins.


Example:

![image](https://user-images.githubusercontent.com/17051931/89961771-6163c900-dc08-11ea-9d71-28de047db1c2.png)




Discussion:
I'm not sure this is a bug rather than a (Python) design choice ("we're all adults"). It does break things for users that intentionally shadow the standard library, though this is not a common use case. 

I definitely agree that this is confusing for new users. How about instead, we warn the user if they use a file name which shadows a standard library?

Did something change in Python 3.7 that you refer to this version?

Resolving patch:
diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py b/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py
--- a/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py
+++ b/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py
@@ -715,10 +715,12 @@ def cell_count(filename=None):
 
 
 # =============================================================================
-# Restoring original PYTHONPATH
+# Extend sys.path with paths that come from Spyder
 # =============================================================================
-try:
-    os.environ['PYTHONPATH'] = os.environ['OLD_PYTHONPATH']
-    del os.environ['OLD_PYTHONPATH']
-except KeyError:
-    pass
+def set_spyder_pythonpath():
+    pypath = os.environ.get('SPY_PYTHONPATH')
+    if pypath:
+        pathlist = pypath.split(os.pathsep)
+        sys.path.extend(pathlist)
+
+set_spyder_pythonpath()
diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py
--- a/spyder/plugins/ipythonconsole/utils/kernelspec.py
+++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py
@@ -24,7 +24,7 @@
 from spyder.utils.conda import (add_quotes, get_conda_activation_script,
                                 get_conda_env_path, is_conda_env)
 from spyder.utils.environ import clean_env
-from spyder.utils.misc import add_pathlist_to_PYTHONPATH, get_python_executable
+from spyder.utils.misc import get_python_executable
 from spyder.utils.programs import is_python_interpreter
 
 # Constants
@@ -129,31 +129,23 @@ def env(self):
         # to the kernel sys.path
         env_vars.pop('VIRTUAL_ENV', None)
 
-        pathlist = CONF.get('main', 'spyder_pythonpath', default=[])
-
-        # Add spyder-kernels subrepo path to pathlist
+        # Add spyder-kernels subrepo path to PYTHONPATH
         if DEV or running_under_pytest():
             repo_path = osp.normpath(osp.join(HERE, '..', '..', '..', '..'))
             subrepo_path = osp.join(repo_path, 'external-deps',
                                     'spyder-kernels')
 
-            if running_under_pytest():
-                # Oddly pathlist is not set as an empty list when running
-                # under pytest
-                pathlist = [subrepo_path]
-            else:
-                pathlist += [subrepo_path] + pathlist
+            env_vars.update({'PYTHONPATH': subrepo_path})
 
-        # Create PYTHONPATH env entry to add it to the kernel
-        pypath = add_pathlist_to_PYTHONPATH([], pathlist, ipyconsole=True,
-                                            drop_env=True)
-
-        # Add our PYTHONPATH to env_vars
-        env_vars.update(pypath)
+        # List of paths declared by the user, plus project's path, to
+        # add to PYTHONPATH
+        pathlist = CONF.get('main', 'spyder_pythonpath', default=[])
+        pypath = os.pathsep.join(pathlist)
 
-        # Environment variables that we need to pass to our sitecustomize
+        # List of modules to exclude from our UMR
         umr_namelist = CONF.get('main_interpreter', 'umr/namelist')
 
+        # Environment variables that we need to pass to the kernel
         env_vars.update({
             'SPY_EXTERNAL_INTERPRETER': not default_interpreter,
             'SPY_UMR_ENABLED': CONF.get('main_interpreter', 'umr/enabled'),
@@ -180,7 +172,8 @@ def env(self):
             'SPY_JEDI_O': CONF.get('ipython_console', 'jedi_completer'),
             'SPY_SYMPY_O': CONF.get('ipython_console', 'symbolic_math'),
             'SPY_TESTING': running_under_pytest() or get_safe_mode(),
-            'SPY_HIDE_CMD': CONF.get('ipython_console', 'hide_cmd_windows')
+            'SPY_HIDE_CMD': CONF.get('ipython_console', 'hide_cmd_windows'),
+            'SPY_PYTHONPATH': pypath
         })
 
         if self.is_pylab is True:


Test patch:
diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
--- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
+++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
@@ -98,6 +98,9 @@ def ipyconsole(qtbot, request):
     """IPython console fixture."""
 
     class MainWindowMock(QMainWindow):
+        def get_spyder_pythonpath(self):
+            return CONF.get('main', 'spyder_pythonpath', [])
+
         def __getattr__(self, attr):
             if attr == 'consoles_menu_actions':
                 return []
@@ -1639,15 +1642,22 @@ def test_kernel_kill(ipyconsole, qtbot):
 
 
 @flaky(max_runs=3)
-def test_wrong_std_module(ipyconsole, qtbot):
+@pytest.mark.parametrize("spyder_pythonpath", [True, False])
+def test_wrong_std_module(ipyconsole, qtbot, tmpdir, spyder_pythonpath):
     """
     Test that a file with the same name of a standard library module in
     the current working directory doesn't break the console.
     """
     # Create an empty file called random.py in the cwd
-    wrong_random_mod = osp.join(os.getcwd(), 'random.py')
-    with open(wrong_random_mod, 'w') as f:
-        f.write('')
+    if spyder_pythonpath:
+        wrong_random_mod = tmpdir.join('random.py')
+        wrong_random_mod.write('')
+        wrong_random_mod = str(wrong_random_mod)
+        CONF.set('main', 'spyder_pythonpath', [str(tmpdir)])
+    else:
+        wrong_random_mod = osp.join(os.getcwd(), 'random.py')
+        with open(wrong_random_mod, 'w') as f:
+            f.write('')
 
     # Create a new client to see if its kernel starts despite the
     # faulty module.
@@ -1658,9 +1668,21 @@ def test_wrong_std_module(ipyconsole, qtbot):
     qtbot.waitUntil(lambda: shell._prompt_html is not None,
                     timeout=SHELL_TIMEOUT)
 
+    # Assert the extra path from spyder_pythonpath was added
+    if spyder_pythonpath:
+        check_sys_path = (
+            "import sys; path_added = r'{}' in sys.path".format(str(tmpdir))
+        )
+        with qtbot.waitSignal(shell.sig_prompt_ready, timeout=30000):
+            shell.execute(check_sys_path)
+        assert shell.get_value('path_added')
+
     # Remove wrong module
     os.remove(wrong_random_mod)
 
+    # Restore CONF
+    CONF.set('main', 'spyder_pythonpath', [])
+
 
 @flaky(max_runs=3)
 @pytest.mark.skipif(os.name == 'nt', reason="no SIGTERM on Windows")