Repository: scrapy/scrapy

Issue:
Add support for bpython.
bpython is a fancy interface to the Python interpreter, with lots of features like syntax highlight (using pygments), autocomplete and pastebin support.

This is a trivial patch, we might rethink the `start_python_console` arguments in order to let the user choose which console the user wants. This code defaults to bpython if it is available, falls back to IPython and, finally, python if no fancier interpreter is around.



Discussion:
+1 for bpython

maybe better to add a setting into .scrapy.cfg so a user could set preferred console there?

@artem-dev yes, I think it'd be great. I thought of adding it to scrapy.cfg but while this makes sense as a project override, it definitely should be set user-wide. I wasn't aware of a .scrapy.cfg though. Cool!

@barraponto Can you fix this PR?

Yeah, I'll give it a try. Never dug into scrapy configurations, but I think it'll be a nice experience.

So I've rebased this and added a scrapy.cfg setting for it. I hope this could get merged now.


Resolving patch:
diff --git a/scrapy/shell.py b/scrapy/shell.py
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -5,6 +5,7 @@
 """
 from __future__ import print_function
 
+import os
 import signal
 import warnings
 
@@ -21,6 +22,8 @@
 from scrapy.utils.console import start_python_console
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.conf import get_config
+from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
 
 
 class Shell(object):
@@ -52,7 +55,29 @@ def start(self, url=None, request=None, response=None, spider=None):
         if self.code:
             print(eval(self.code, globals(), self.vars))
         else:
-            start_python_console(self.vars)
+            """
+            Detect interactive shell setting in scrapy.cfg
+            e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg
+            [settings]
+            # shell can be one of ipython, bpython or python;
+            # to be used as the interactive python console, if available.
+            # (default is ipython, fallbacks in the order listed above)
+            shell = python
+            """
+            cfg = get_config()
+            section, option = 'settings', 'shell'
+            env = os.environ.get('SCRAPY_PYTHON_SHELL')
+            shells = []
+            if env:
+                shells += env.strip().lower().split(',')
+            elif cfg.has_option(section, option):
+                shells += [cfg.get(section, option).strip().lower()]
+            else:  # try all by default
+                shells += DEFAULT_PYTHON_SHELLS.keys()
+            # always add standard shell as fallback
+            shells += ['python']
+            start_python_console(self.vars, shells=shells,
+                                 banner=self.vars.pop('banner', ''))
 
     def _schedule(self, request, spider):
         spider = self._open_spider(request, spider)
@@ -103,21 +128,25 @@ def populate_vars(self, response=None, request=None, spider=None):
         self.vars['shelp'] = self.print_help
         self.update_vars(self.vars)
         if not self.code:
-            self.print_help()
+            self.vars['banner'] = self.get_help()
 
     def print_help(self):
-        self.p("Available Scrapy objects:")
+        print(self.get_help())
+
+    def get_help(self):
+        b = []
+        b.append("Available Scrapy objects:")
         for k, v in sorted(self.vars.items()):
             if self._is_relevant(v):
-                self.p("  %-10s %s" % (k, v))
-        self.p("Useful shortcuts:")
-        self.p("  shelp()           Shell help (print this help)")
+                b.append("  %-10s %s" % (k, v))
+        b.append("Useful shortcuts:")
+        b.append("  shelp()           Shell help (print this help)")
         if self.inthread:
-            self.p("  fetch(req_or_url) Fetch request (or URL) and update local objects")
-        self.p("  view(response)    View response in a browser")
+            b.append("  fetch(req_or_url) Fetch request (or URL) and "
+                     "update local objects")
+        b.append("  view(response)    View response in a browser")
 
-    def p(self, line=''):
-        print("[s] %s" % line)
+        return "\n".join(["[s] %s" % l for l in b])
 
     def _is_relevant(self, value):
         return isinstance(value, self.relevant_classes)
@@ -141,6 +170,7 @@ def _request_deferred(request):
     """
     request_callback = request.callback
     request_errback = request.errback
+
     def _restore_callbacks(result):
         request.callback = request_callback
         request.errback = request_errback
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,37 +1,79 @@
+from functools import wraps
+from collections import OrderedDict
 
-def start_python_console(namespace=None, noipython=False, banner=''):
-    """Start Python console binded to the given namespace. If IPython is
-    available, an IPython console will be started instead, unless `noipython`
-    is True. Also, tab completion will be used on Unix systems.
+def _embed_ipython_shell(namespace={}, banner=''):
+    """Start an IPython Shell"""
+    try:
+        from IPython.terminal.embed import InteractiveShellEmbed
+        from IPython.terminal.ipapp import load_default_config
+    except ImportError:
+        from IPython.frontend.terminal.embed import InteractiveShellEmbed
+        from IPython.frontend.terminal.ipapp import load_default_config
+
+    @wraps(_embed_ipython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        config = load_default_config()
+        shell = InteractiveShellEmbed(
+            banner1=banner, user_ns=namespace, config=config)
+        shell()
+    return wrapper
+
+def _embed_bpython_shell(namespace={}, banner=''):
+    """Start a bpython shell"""
+    import bpython
+    @wraps(_embed_bpython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        bpython.embed(locals_=namespace, banner=banner)
+    return wrapper
+
+def _embed_standard_shell(namespace={}, banner=''):
+    """Start a standard python shell"""
+    import code
+    try: # readline module is only available on unix systems
+        import readline
+    except ImportError:
+        pass
+    else:
+        import rlcompleter
+        readline.parse_and_bind("tab:complete")
+    @wraps(_embed_standard_shell)
+    def wrapper(namespace=namespace, banner=''):
+        code.interact(banner=banner, local=namespace)
+    return wrapper
+
+DEFAULT_PYTHON_SHELLS = OrderedDict([
+    ('ipython', _embed_ipython_shell),
+    ('bpython', _embed_bpython_shell),
+    ( 'python', _embed_standard_shell),
+])
+
+def get_shell_embed_func(shells=None, known_shells=None):
+    """Return the first acceptable shell-embed function
+    from a given list of shell names.
+    """
+    if shells is None: # list, preference order of shells
+        shells = DEFAULT_PYTHON_SHELLS.keys()
+    if known_shells is None: # available embeddable shells
+        known_shells = DEFAULT_PYTHON_SHELLS.copy()
+    for shell in shells:
+        if shell in known_shells:
+            try:
+                # function test: run all setup code (imports),
+                # but dont fall into the shell
+                return known_shells[shell]()
+            except ImportError:
+                continue
+
+def start_python_console(namespace=None, banner='', shells=None):
+    """Start Python console bound to the given namespace.
+    Readline support and tab completion will be used on Unix, if available.
     """
     if namespace is None:
         namespace = {}
 
     try:
-        try: # use IPython if available
-            if noipython:
-                raise ImportError()
-
-            try:
-                from IPython.terminal.embed import InteractiveShellEmbed
-                from IPython.terminal.ipapp import load_default_config
-            except ImportError:
-                from IPython.frontend.terminal.embed import InteractiveShellEmbed
-                from IPython.frontend.terminal.ipapp import load_default_config
-
-            config = load_default_config()
-            shell = InteractiveShellEmbed(
-                banner1=banner, user_ns=namespace, config=config)
-            shell()
-        except ImportError:
-            import code
-            try: # readline module is only available on unix systems
-                import readline
-            except ImportError:
-                pass
-            else:
-                import rlcompleter
-                readline.parse_and_bind("tab:complete")
-            code.interact(banner=banner, local=namespace)
+        shell = get_shell_embed_func(shells)
+        if shell is not None:
+            shell(namespace=namespace, banner=banner)
     except SystemExit: # raised when using exit() in python code.interact
         pass


Test patch:
diff --git a/tests/requirements.txt b/tests/requirements.txt
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,3 +6,6 @@ pytest-twisted
 pytest-cov
 jmespath
 testfixtures
+# optional for shell wrapper tests
+bpython
+ipython
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
new file mode 100644
--- /dev/null
+++ b/tests/test_utils_console.py
@@ -0,0 +1,44 @@
+import unittest
+
+from scrapy.utils.console import get_shell_embed_func
+try:
+    import bpython
+    bpy = True
+    del bpython
+except ImportError:
+    bpy = False
+try:
+    import IPython
+    ipy = True
+    del IPython
+except ImportError:
+    ipy = False
+
+class UtilsConsoleTestCase(unittest.TestCase):
+
+    def test_get_shell_embed_func(self):
+
+        shell = get_shell_embed_func(['invalid'])
+        self.assertEqual(shell, None)
+
+        shell = get_shell_embed_func(['invalid','python'])
+        self.assertTrue(callable(shell))
+        self.assertEqual(shell.__name__, '_embed_standard_shell')
+
+    @unittest.skipIf(not bpy, 'bpython not available in testenv')
+    def test_get_shell_embed_func2(self):
+
+        shell = get_shell_embed_func(['bpython'])
+        self.assertTrue(callable(shell))
+        self.assertEqual(shell.__name__, '_embed_bpython_shell')
+
+    @unittest.skipIf(not ipy, 'IPython not available in testenv')
+    def test_get_shell_embed_func3(self):
+
+        # default shell should be 'ipython'
+        shell = get_shell_embed_func()
+        self.assertEqual(shell.__name__, '_embed_ipython_shell')
+
+
+if __name__ == "__main__":
+    unittest.main()