Repository: scrapy/scrapy

Issue:
Add note that CrawlerProcess.crawl() needs a crawler or spider class, not a spider instance
Even if [it's in the docs](http://doc.scrapy.org/en/latest/topics/api.html#scrapy.crawler.CrawlerProcess.crawl), 

> crawler_or_spidercls (Crawler instance, Spider subclass or string) – already created crawler, or a spider class or spider’s name inside the project to create it

users still miss sometimes that `.crawl()` expects a Spider class (or a crawler) but not a Spider instance.

e.g. http://stackoverflow.com/q/39639568

> I overlooked the fact that the class itself has to be passed as an argument, and not an instance of that class.
> This is quite subtle on the eye but makes all the difference

This ticket is to make it a bit more prominent.



Discussion:

Resolving patch:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -7,6 +7,7 @@
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
+from scrapy import Spider
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
 from scrapy.interfaces import ISpiderLoader
@@ -27,6 +28,10 @@
 class Crawler(object):
 
     def __init__(self, spidercls, settings=None):
+        if isinstance(spidercls, Spider):
+            raise ValueError(
+                'The spidercls argument must be a class, not an object')
+
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
@@ -168,6 +173,10 @@ def crawl(self, crawler_or_spidercls, *args, **kwargs):
 
         :param dict kwargs: keyword arguments to initialize the spider
         """
+        if isinstance(crawler_or_spidercls, Spider):
+            raise ValueError(
+                'The crawler_or_spidercls argument cannot be a spider object, '
+                'it must be a spider class (or a Crawler object)')
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
@@ -195,6 +204,10 @@ def create_crawler(self, crawler_or_spidercls):
           a spider with this name in a Scrapy project (using spider loader),
           then creates a Crawler instance for it.
         """
+        if isinstance(crawler_or_spidercls, Spider):
+            raise ValueError(
+                'The crawler_or_spidercls argument cannot be a spider object, '
+                'it must be a spider class (or a Crawler object)')
         if isinstance(crawler_or_spidercls, Crawler):
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)


Test patch:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -4,6 +4,7 @@
 
 from twisted.internet import defer
 from twisted.trial import unittest
+from pytest import raises
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@@ -66,6 +67,10 @@ def test_crawler_accepts_None(self):
         crawler = Crawler(DefaultSpider)
         self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
+    def test_crawler_rejects_spider_objects(self):
+        with raises(ValueError):
+            Crawler(DefaultSpider())
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):
@@ -177,6 +182,14 @@ def test_spidermanager_deprecation(self):
             self.assertEqual(len(w), 1)
             self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
 
+    def test_crawl_rejects_spider_objects(self):
+        with raises(ValueError):
+            CrawlerRunner().crawl(DefaultSpider())
+
+    def test_create_crawler_rejects_spider_objects(self):
+        with raises(ValueError):
+            CrawlerRunner().create_crawler(DefaultSpider())
+
 
 class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -25,7 +25,7 @@ def tearDown(self):
 
     def test_not_enabled(self):
         settings = Settings({'HTTPPROXY_ENABLED': False})
-        crawler = Crawler(spider, settings)
+        crawler = Crawler(Spider, settings)
         self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
 
     def test_no_environment_proxies(self):