Repository: scrapy/scrapy
Pull request #1528 (2015-10-06 12:36:54)

public Crawler.create_crawler method
CrawlerRunner.crawl returns a Deferred, but in all cases I used this method in addition to waiting for crawl to finish it was necessary to connect signal handlers, and you need Crawler instance for that. So 
```python
d = runner.crawl('spidername')  # or runner.create_crawler(spider_cls)
```
was not enough, I wanted

```python
crawler = runner.create_crawler('spidername')  # or runner.create_crawler(spider_cls)
crawler.signals.connect(...)
d = runner.crawl(crawler)
```

This means it was necessary to copy-paste the same code 
```python
crawler = crawler_or_spidercls
if not isinstance(crawler_or_spidercls, Crawler):
    crawler = runner._create_crawler(crawler_or_spidercls)
```
in all CrawlerRunner subclasses / wrappers, and using a private `_create_crawler` method. What about adding a public method for it? As a bonus point, we won't be using a private method in `scrapy.utils.test.get_crawler`.



codecov-io: ## [Current coverage][1] is `82.51%`
> Merging **#1528** into **master** will increase coverage by **+0.04%** as of [`e7decb9`][3]



[1]: https://codecov.io/github/scrapy/scrapy?ref=e7decb9cd1ab571b00c1ad184362f021ae38ac0f
[2]: https://codecov.io/github/scrapy/scrapy/features/suggestions?ref=e7decb9cd1ab571b00c1ad184362f021ae38ac0f
[3]: https://codecov.io/github/scrapy/scrapy/commit/e7decb9cd1ab571b00c1ad184362f021ae38ac0f
[4]: https://codecov.io/github/scrapy/scrapy/compare/9676cf8e53affbc9f2dc95a8c3e14601e8e2bfdb...e7decb9cd1ab571b00c1ad184362f021ae38ac0f

> Powered by [Codecov](https://codecov.io). Updated on successful CI builds.

Patch:
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 11c0aadb6..a71b8bcee 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -36,7 +36,7 @@ by a string: the full Python path to the extension's class name. For example::
 
     EXTENSIONS = {
         'scrapy.extensions.corestats.CoreStats': 500,
-        'scrapy.extensions.telnet.TelnetConsole': 500,
+        'scrapy.telnet.TelnetConsole': 500,
     }
 
 
@@ -179,10 +179,10 @@ enabled (see :ref:`topics-stats`).
 Telnet console extension
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.extensions.telnet
+.. module:: scrapy.telnet
    :synopsis: Telnet console
 
-.. class:: scrapy.extensions.telnet.TelnetConsole
+.. class:: scrapy.telnet.TelnetConsole
 
 Provides a telnet console for getting into a Python interpreter inside the
 currently running Scrapy process, which can be very useful for debugging.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8908fae7e..642f4eb84 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -548,7 +548,7 @@ Default::
 
     {
         'scrapy.extensions.corestats.CoreStats': 0,
-        'scrapy.extensions.telnet.TelnetConsole': 0,
+        'scrapy.telnet.TelnetConsole': 0,
         'scrapy.extensions.memusage.MemoryUsage': 0,
         'scrapy.extensions.memdebug.MemoryDebugger': 0,
         'scrapy.extensions.closespider.CloseSpider': 0,
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index ce79c9f35..b1ff88771 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -4,7 +4,7 @@
 Telnet Console
 ==============
 
-.. module:: scrapy.extensions.telnet
+.. module:: scrapy.telnet
    :synopsis: The Telnet Console
 
 Scrapy comes with a built-in telnet console for inspecting and controlling a
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a1937f2bd..bdcfa9d0c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -148,9 +148,7 @@ class CrawlerRunner(object):
 
         :param dict kwargs: keyword arguments to initialize the spider
         """
-        crawler = crawler_or_spidercls
-        if not isinstance(crawler_or_spidercls, Crawler):
-            crawler = self._create_crawler(crawler_or_spidercls)
+        crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
     def _crawl(self, crawler, *args, **kwargs):
@@ -165,6 +163,21 @@ class CrawlerRunner(object):
 
         return d.addBoth(_done)
 
+    def create_crawler(self, crawler_or_spidercls):
+        """
+        Return a :class:`~scrapy.crawler.Crawler` object.
+
+        * If `crawler_or_spidercls` is a Crawler, it is returned as-is.
+        * If `crawler_or_spidercls` is a Spider subclass, a new Crawler
+          is constructed for it.
+        * If `crawler_or_spidercls` is a string, this function finds
+          a spider with this name in a Scrapy project (using spider loader),
+          then creates a Crawler instance for it.
+        """
+        if isinstance(crawler_or_spidercls, Crawler):
+            return crawler_or_spidercls
+        return self._create_crawler(crawler_or_spidercls)
+
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spider_loader.load(spidercls)
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
deleted file mode 100644
index d9add1d97..000000000
--- a/scrapy/extensions/telnet.py
+++ /dev/null
@@ -1,88 +0,0 @@
-"""
-Scrapy Telnet Console extension
-
-See documentation in docs/topics/telnetconsole.rst
-"""
-
-import pprint
-import logging
-
-from twisted.internet import protocol
-try:
-    from twisted.conch import manhole, telnet
-    from twisted.conch.insults import insults
-    TWISTED_CONCH_AVAILABLE = True
-except ImportError:
-    TWISTED_CONCH_AVAILABLE = False
-
-from scrapy.exceptions import NotConfigured
-from scrapy import signals
-from scrapy.utils.trackref import print_live_refs
-from scrapy.utils.engine import print_engine_status
-from scrapy.utils.reactor import listen_tcp
-
-try:
-    import guppy
-    hpy = guppy.hpy()
-except ImportError:
-    hpy = None
-
-logger = logging.getLogger(__name__)
-
-# signal to update telnet variables
-# args: telnet_vars
-update_telnet_vars = object()
-
-
-class TelnetConsole(protocol.ServerFactory):
-
-    def __init__(self, crawler):
-        if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
-            raise NotConfigured
-        if not TWISTED_CONCH_AVAILABLE:
-            raise NotConfigured
-        self.crawler = crawler
-        self.noisy = False
-        self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
-        self.host = crawler.settings['TELNETCONSOLE_HOST']
-        self.crawler.signals.connect(self.start_listening, signals.engine_started)
-        self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    def start_listening(self):
-        self.port = listen_tcp(self.portrange, self.host, self)
-        h = self.port.getHost()
-        logger.debug("Telnet console listening on %(host)s:%(port)d",
-                     {'host': h.host, 'port': h.port},
-                     extra={'crawler': self.crawler})
-
-    def stop_listening(self):
-        self.port.stopListening()
-
-    def protocol(self):
-        telnet_vars = self._get_telnet_vars()
-        return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
-            insults.ServerProtocol, manhole.Manhole, telnet_vars)
-
-    def _get_telnet_vars(self):
-        # Note: if you add entries here also update topics/telnetconsole.rst
-        telnet_vars = {
-            'engine': self.crawler.engine,
-            'spider': self.crawler.engine.spider,
-            'slot': self.crawler.engine.slot,
-            'crawler': self.crawler,
-            'extensions': self.crawler.extensions,
-            'stats': self.crawler.stats,
-            'settings': self.crawler.settings,
-            'est': lambda: print_engine_status(self.crawler.engine),
-            'p': pprint.pprint,
-            'prefs': print_live_refs,
-            'hpy': hpy,
-            'help': "This is Scrapy telnet console. For more info see: " \
-                "http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
-        }
-        self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
-        return telnet_vars
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 375efcdbb..ba7cece19 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -115,7 +115,7 @@ except KeyError:
 
 EXTENSIONS = {
     'scrapy.extensions.corestats.CoreStats': 0,
-    'scrapy.extensions.telnet.TelnetConsole': 0,
+    'scrapy.telnet.TelnetConsole': 0,
     'scrapy.extensions.memusage.MemoryUsage': 0,
     'scrapy.extensions.memdebug.MemoryDebugger': 0,
     'scrapy.extensions.closespider.CloseSpider': 0,
diff --git a/scrapy/telnet.py b/scrapy/telnet.py
index 70915f486..d9add1d97 100644
--- a/scrapy/telnet.py
+++ b/scrapy/telnet.py
@@ -1,7 +1,88 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.telnet` is deprecated, "
-              "use `scrapy.extensions.telnet` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
+"""
+Scrapy Telnet Console extension
 
-from scrapy.extensions.telnet import *
+See documentation in docs/topics/telnetconsole.rst
+"""
+
+import pprint
+import logging
+
+from twisted.internet import protocol
+try:
+    from twisted.conch import manhole, telnet
+    from twisted.conch.insults import insults
+    TWISTED_CONCH_AVAILABLE = True
+except ImportError:
+    TWISTED_CONCH_AVAILABLE = False
+
+from scrapy.exceptions import NotConfigured
+from scrapy import signals
+from scrapy.utils.trackref import print_live_refs
+from scrapy.utils.engine import print_engine_status
+from scrapy.utils.reactor import listen_tcp
+
+try:
+    import guppy
+    hpy = guppy.hpy()
+except ImportError:
+    hpy = None
+
+logger = logging.getLogger(__name__)
+
+# signal to update telnet variables
+# args: telnet_vars
+update_telnet_vars = object()
+
+
+class TelnetConsole(protocol.ServerFactory):
+
+    def __init__(self, crawler):
+        if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
+            raise NotConfigured
+        if not TWISTED_CONCH_AVAILABLE:
+            raise NotConfigured
+        self.crawler = crawler
+        self.noisy = False
+        self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
+        self.host = crawler.settings['TELNETCONSOLE_HOST']
+        self.crawler.signals.connect(self.start_listening, signals.engine_started)
+        self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def start_listening(self):
+        self.port = listen_tcp(self.portrange, self.host, self)
+        h = self.port.getHost()
+        logger.debug("Telnet console listening on %(host)s:%(port)d",
+                     {'host': h.host, 'port': h.port},
+                     extra={'crawler': self.crawler})
+
+    def stop_listening(self):
+        self.port.stopListening()
+
+    def protocol(self):
+        telnet_vars = self._get_telnet_vars()
+        return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
+            insults.ServerProtocol, manhole.Manhole, telnet_vars)
+
+    def _get_telnet_vars(self):
+        # Note: if you add entries here also update topics/telnetconsole.rst
+        telnet_vars = {
+            'engine': self.crawler.engine,
+            'spider': self.crawler.engine.spider,
+            'slot': self.crawler.engine.slot,
+            'crawler': self.crawler,
+            'extensions': self.crawler.extensions,
+            'stats': self.crawler.stats,
+            'settings': self.crawler.settings,
+            'est': lambda: print_engine_status(self.crawler.engine),
+            'p': pprint.pprint,
+            'prefs': print_live_refs,
+            'hpy': hpy,
+            'help': "This is Scrapy telnet console. For more info see: " \
+                "http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
+        }
+        self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
+        return telnet_vars
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 822812c9a..2e660d7cf 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -56,7 +56,7 @@ NEWSPIDER_MODULE = '$project_name.spiders'
 # Enable or disable extensions
 # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
 #EXTENSIONS = {
-#    'scrapy.extensions.telnet.TelnetConsole': None,
+#    'scrapy.telnet.TelnetConsole': None,
 #}
 
 # Configure item pipelines
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 0fc33e0c4..37e94fae9 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -138,7 +138,6 @@ DEPRECATION_RULES = [
     ('scrapy.command.', 'scrapy.commands.'),
     ('scrapy.dupefilter.', 'scrapy.dupefilters.'),
     ('scrapy.linkextractor.', 'scrapy.linkextractors.'),
-    ('scrapy.telnet.', 'scrapy.extensions.telnet.'),
     ('scrapy.spider.', 'scrapy.spiders.'),
     ('scrapy.squeue.', 'scrapy.squeues.'),
     ('scrapy.statscol.', 'scrapy.statscollectors.'),


Test patch:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index bec9bdda9..51edfd353 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -26,11 +26,10 @@ def get_crawler(spidercls=None, settings_dict=None):
     priority.
     """
     from scrapy.crawler import CrawlerRunner
-    from scrapy.settings import Settings
     from scrapy.spiders import Spider
 
-    runner = CrawlerRunner(Settings(settings_dict))
-    return runner._create_crawler(spidercls or Spider)
+    runner = CrawlerRunner(settings_dict)
+    return runner.create_crawler(spidercls or Spider)
 
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index b2105dcfa..814eb30d2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -6,12 +6,12 @@ from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
-from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
 from tests import mock
 from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
-from scrapy.http import Request
 
 
 class CrawlTestCase(TestCase):
@@ -19,13 +19,14 @@ class CrawlTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_follow_all(self):
-        crawler = get_crawler(FollowAllSpider)
+        crawler = self.runner.create_crawler(FollowAllSpider)
         yield crawler.crawl()
         self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
@@ -41,7 +42,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def _test_delay(self, delay, randomize):
         settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
-        crawler = get_crawler(FollowAllSpider, settings)
+        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
         yield crawler.crawl(maxlatency=delay * 2)
         t = crawler.spider.times
         totaltime = t[-1] - t[0]
@@ -52,7 +53,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
-        crawler = get_crawler(DelaySpider)
+        crawler = self.runner.create_crawler(DelaySpider)
         yield crawler.crawl(n=0.5)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 > 0)
@@ -60,7 +61,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_failure(self):
-        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
+        crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
         yield crawler.crawl(n=0.5)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -75,14 +76,14 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_retry_503(self):
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/status?n=503")
         self._assert_retried(l)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:65432/status?n=503")
         self._assert_retried(l)
@@ -91,7 +92,7 @@ class CrawlTestCase(TestCase):
     def test_retry_dns_error(self):
         with mock.patch('socket.gethostbyname',
                         side_effect=socket.gaierror(-5, 'No address associated with hostname')):
-            crawler = get_crawler(SimpleSpider)
+            crawler = self.runner.create_crawler(SimpleSpider)
             with LogCapture() as l:
                 yield crawler.crawl("http://example.com/")
             self._assert_retried(l)
@@ -99,7 +100,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
         with LogCapture('scrapy', level=logging.ERROR) as l:
-            crawler = get_crawler(BrokenStartRequestsSpider)
+            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1)
 
         self.assertEqual(len(l.records), 1)
@@ -110,7 +111,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
         with LogCapture('scrapy', level=logging.ERROR) as l:
-            crawler = get_crawler(BrokenStartRequestsSpider)
+            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1)
 
         self.assertEqual(len(l.records), 1)
@@ -121,7 +122,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_lazyness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = get_crawler(BrokenStartRequestsSpider, settings)
+        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
         yield crawler.crawl()
         #self.assertTrue(False, crawler.spider.seedsseen)
         #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@@ -130,7 +131,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
+        crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
         yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3)
         self.assertEqual(crawler.spider.visited, 6)
 
@@ -159,7 +160,7 @@ Connection: close
 foo body
 with multiples lines
 '''})
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
         self.assertEqual(str(l).count("Got response 200"), 1)
@@ -167,7 +168,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/drop?abort=0")
         self._assert_retried(l)
@@ -175,7 +176,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
-        crawler = get_crawler(SimpleSpider)
+        crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as l:
             yield crawler.crawl("http://localhost:8998/drop?abort=1")
         self._assert_retried(l)
@@ -194,7 +195,7 @@ with multiples lines
         req0.meta['next'] = req1
         req1.meta['next'] = req2
         req2.meta['next'] = req3
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = self.runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=req0)
         # basic asserts in case of weird communication errors
         self.assertIn('responses', crawler.spider.meta)
@@ -220,7 +221,7 @@ with multiples lines
         def cb(response):
             est.append(get_engine_status(crawler.engine))
 
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = self.runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb)
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
@@ -244,6 +245,24 @@ with multiples lines
             def start_requests(self):
                 raise TestError
 
-        crawler = get_crawler(FaultySpider)
+        crawler = self.runner.create_crawler(FaultySpider)
         yield self.assertFailure(crawler.crawl(), TestError)
         self.assertFalse(crawler.crawling)
+
+    @defer.inlineCallbacks
+    def test_crawlerrunner_accepts_crawler(self):
+        crawler = self.runner.create_crawler(SimpleSpider)
+        with LogCapture() as log:
+            yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200")
+        self.assertIn("Got response 200", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawl_multiple(self):
+        self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200")
+        self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503")
+
+        with LogCapture() as log:
+            yield self.runner.join()
+
+        self._assert_retried(log)
+        self.assertIn("Got response 200", str(log))
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 7cb5e299b..68dca2e98 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -8,10 +8,12 @@ from twisted.trial import unittest
 
 # ugly hack to avoid cyclic imports of scrapy.spiders when running this test
 # alone
+import scrapy
 from scrapy.interfaces import ISpiderLoader
 from scrapy.spiderloader import SpiderLoader
 from scrapy.settings import Settings
 from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
 
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -76,3 +78,14 @@ class SpiderLoaderTest(unittest.TestCase):
         settings = Settings({'SPIDER_MODULES': [module]})
         self.spider_loader = SpiderLoader.from_settings(settings)
         assert len(self.spider_loader._spiders) == 0
+
+    def test_crawler_runner_loading(self):
+        module = 'tests.test_spiderloader.test_spiders.spider1'
+        runner = CrawlerRunner({'SPIDER_MODULES': [module]})
+
+        self.assertRaisesRegexp(KeyError, 'Spider not found',
+                                runner.create_crawler, 'spider2')
+
+        crawler = runner.create_crawler('spider1')
+        self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
+        self.assertEqual(crawler.spidercls.name, 'spider1')