Repository: scrapy/scrapy
Pull request #1444 (2015-08-21 00:15:10)

[MRG +1] bpython support
Related to #1100, this shows the banner correctly on standard python, ipython and bpython.


Patch:
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 52c4fe446..16af52eea 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -16,6 +16,8 @@ accepts a different set of arguments and options.
 (The ``scrapy deploy`` command has been removed in 1.0 in favor of the
 standalone ``scrapyd-deploy``. See `Deploying your project`_.)
 
+.. _topics-config-settings:
+
 Configuration settings
 ======================
 
@@ -34,8 +36,9 @@ and project-wide settings will override all others, when defined.
 Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
 
-* ``SCRAPY_SETTINGS_MODULE`` (See :ref:`topics-settings-module-envvar`)
+* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
 * ``SCRAPY_PROJECT``
+* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
 
 .. _topics-project-structure:
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 9c9411d6d..327e22e90 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -17,6 +17,9 @@ spider, without having to run the spider to test every change.
 Once you get familiarized with the Scrapy shell, you'll see that it's an
 invaluable tool for developing and debugging your spiders.
 
+Configuring the shell
+=====================
+
 If you have `IPython`_ installed, the Scrapy shell will use it (instead of the
 standard Python console). The `IPython`_ console is much more powerful and
 provides smart auto-completion and colorized output, among other things.
@@ -25,8 +28,20 @@ We highly recommend you install `IPython`_, specially if you're working on
 Unix systems (where `IPython`_ excels). See the `IPython installation guide`_
 for more info.
 
+Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
+is unavailable.
+
+Through scrapy's settings you can configure it to use any one of
+``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
+are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
+variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
+
+    [settings]
+    shell = bpython
+
 .. _IPython: http://ipython.org/
 .. _IPython installation guide: http://ipython.org/install.html
+.. _bpython: http://www.bpython-interpreter.org/
 
 Launch the shell
 ================
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 4142396ab..f008ce39a 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -5,6 +5,7 @@ See documentation in docs/topics/shell.rst
 """
 from __future__ import print_function
 
+import os
 import signal
 import warnings
 
@@ -21,6 +22,8 @@ from scrapy.spiders import Spider
 from scrapy.utils.console import start_python_console
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.conf import get_config
+from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 
 class Shell(object):
@@ -52,7 +55,29 @@ class Shell(object):
         if self.code:
             print(eval(self.code, globals(), self.vars))
         else:
-            start_python_console(self.vars)
+            """
+            Detect interactive shell setting in scrapy.cfg
+            e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg
+            [settings]
+            # shell can be one of ipython, bpython or python;
+            # to be used as the interactive python console, if available.
+            # (default is ipython, fallbacks in the order listed above)
+            shell = python
+            """
+            cfg = get_config()
+            section, option = 'settings', 'shell'
+            env = os.environ.get('SCRAPY_PYTHON_SHELL')
+            shells = []
+            if env:
+                shells += env.strip().lower().split(',')
+            elif cfg.has_option(section, option):
+                shells += [cfg.get(section, option).strip().lower()]
+            else:  # try all by default
+                shells += DEFAULT_PYTHON_SHELLS.keys()
+            # always add standard shell as fallback
+            shells += ['python']
+            start_python_console(self.vars, shells=shells,
+                                 banner=self.vars.pop('banner', ''))
 
     def _schedule(self, request, spider):
         spider = self._open_spider(request, spider)
@@ -103,21 +128,25 @@ class Shell(object):
         self.vars['shelp'] = self.print_help
         self.update_vars(self.vars)
         if not self.code:
-            self.print_help()
+            self.vars['banner'] = self.get_help()
 
     def print_help(self):
-        self.p("Available Scrapy objects:")
+        print(self.get_help())
+
+    def get_help(self):
+        b = []
+        b.append("Available Scrapy objects:")
         for k, v in sorted(self.vars.items()):
             if self._is_relevant(v):
-                self.p("  %-10s %s" % (k, v))
-        self.p("Useful shortcuts:")
-        self.p("  shelp()           Shell help (print this help)")
+                b.append("  %-10s %s" % (k, v))
+        b.append("Useful shortcuts:")
+        b.append("  shelp()           Shell help (print this help)")
         if self.inthread:
-            self.p("  fetch(req_or_url) Fetch request (or URL) and update local objects")
-        self.p("  view(response)    View response in a browser")
+            b.append("  fetch(req_or_url) Fetch request (or URL) and "
+                     "update local objects")
+        b.append("  view(response)    View response in a browser")
 
-    def p(self, line=''):
-        print("[s] %s" % line)
+        return "\n".join(["[s] %s" % l for l in b])
 
     def _is_relevant(self, value):
         return isinstance(value, self.relevant_classes)
@@ -141,6 +170,7 @@ def _request_deferred(request):
     """
     request_callback = request.callback
     request_errback = request.errback
+
     def _restore_callbacks(result):
         request.callback = request_callback
         request.errback = request_errback
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 924501002..a712df30e 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,37 +1,79 @@
+from functools import wraps
+from collections import OrderedDict
 
-def start_python_console(namespace=None, noipython=False, banner=''):
-    """Start Python console binded to the given namespace. If IPython is
-    available, an IPython console will be started instead, unless `noipython`
-    is True. Also, tab completion will be used on Unix systems.
+def _embed_ipython_shell(namespace={}, banner=''):
+    """Start an IPython Shell"""
+    try:
+        from IPython.terminal.embed import InteractiveShellEmbed
+        from IPython.terminal.ipapp import load_default_config
+    except ImportError:
+        from IPython.frontend.terminal.embed import InteractiveShellEmbed
+        from IPython.frontend.terminal.ipapp import load_default_config
+
+    @wraps(_embed_ipython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        config = load_default_config()
+        shell = InteractiveShellEmbed(
+            banner1=banner, user_ns=namespace, config=config)
+        shell()
+    return wrapper
+
+def _embed_bpython_shell(namespace={}, banner=''):
+    """Start a bpython shell"""
+    import bpython
+    @wraps(_embed_bpython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        bpython.embed(locals_=namespace, banner=banner)
+    return wrapper
+
+def _embed_standard_shell(namespace={}, banner=''):
+    """Start a standard python shell"""
+    import code
+    try: # readline module is only available on unix systems
+        import readline
+    except ImportError:
+        pass
+    else:
+        import rlcompleter
+        readline.parse_and_bind("tab:complete")
+    @wraps(_embed_standard_shell)
+    def wrapper(namespace=namespace, banner=''):
+        code.interact(banner=banner, local=namespace)
+    return wrapper
+
+DEFAULT_PYTHON_SHELLS = OrderedDict([
+    ('ipython', _embed_ipython_shell),
+    ('bpython', _embed_bpython_shell),
+    ( 'python', _embed_standard_shell),
+])
+
+def get_shell_embed_func(shells=None, known_shells=None):
+    """Return the first acceptable shell-embed function
+    from a given list of shell names.
+    """
+    if shells is None: # list, preference order of shells
+        shells = DEFAULT_PYTHON_SHELLS.keys()
+    if known_shells is None: # available embeddable shells
+        known_shells = DEFAULT_PYTHON_SHELLS.copy()
+    for shell in shells:
+        if shell in known_shells:
+            try:
+                # function test: run all setup code (imports),
+                # but dont fall into the shell
+                return known_shells[shell]()
+            except ImportError:
+                continue
+
+def start_python_console(namespace=None, banner='', shells=None):
+    """Start Python console bound to the given namespace.
+    Readline support and tab completion will be used on Unix, if available.
     """
     if namespace is None:
         namespace = {}
 
     try:
-        try: # use IPython if available
-            if noipython:
-                raise ImportError()
-
-            try:
-                from IPython.terminal.embed import InteractiveShellEmbed
-                from IPython.terminal.ipapp import load_default_config
-            except ImportError:
-                from IPython.frontend.terminal.embed import InteractiveShellEmbed
-                from IPython.frontend.terminal.ipapp import load_default_config
-
-            config = load_default_config()
-            shell = InteractiveShellEmbed(
-                banner1=banner, user_ns=namespace, config=config)
-            shell()
-        except ImportError:
-            import code
-            try: # readline module is only available on unix systems
-                import readline
-            except ImportError:
-                pass
-            else:
-                import rlcompleter
-                readline.parse_and_bind("tab:complete")
-            code.interact(banner=banner, local=namespace)
+        shell = get_shell_embed_func(shells)
+        if shell is not None:
+            shell(namespace=namespace, banner=banner)
     except SystemExit: # raised when using exit() in python code.interact
         pass


Test patch:
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 452a884e0..f88d17538 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,3 +6,6 @@ pytest-twisted
 pytest-cov
 jmespath
 testfixtures
+# optional for shell wrapper tests
+bpython
+ipython
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
new file mode 100644
index 000000000..65782747b
--- /dev/null
+++ b/tests/test_utils_console.py
@@ -0,0 +1,44 @@
+import unittest
+
+from scrapy.utils.console import get_shell_embed_func
+try:
+    import bpython
+    bpy = True
+    del bpython
+except ImportError:
+    bpy = False
+try:
+    import IPython
+    ipy = True
+    del IPython
+except ImportError:
+    ipy = False
+
+class UtilsConsoleTestCase(unittest.TestCase):
+
+    def test_get_shell_embed_func(self):
+
+        shell = get_shell_embed_func(['invalid'])
+        self.assertEqual(shell, None)
+
+        shell = get_shell_embed_func(['invalid','python'])
+        self.assertTrue(callable(shell))
+        self.assertEqual(shell.__name__, '_embed_standard_shell')
+
+    @unittest.skipIf(not bpy, 'bpython not available in testenv')
+    def test_get_shell_embed_func2(self):
+
+        shell = get_shell_embed_func(['bpython'])
+        self.assertTrue(callable(shell))
+        self.assertEqual(shell.__name__, '_embed_bpython_shell')
+
+    @unittest.skipIf(not ipy, 'IPython not available in testenv')
+    def test_get_shell_embed_func3(self):
+
+        # default shell should be 'ipython'
+        shell = get_shell_embed_func()
+        self.assertEqual(shell.__name__, '_embed_ipython_shell')
+
+
+if __name__ == "__main__":
+    unittest.main()