Repository: scrapy/scrapy

Issue:
Multiple crawls with the same crawler
While we don't support using the same crawler for multiple spiders, it is currently possible to run the same crawler (with the same spider) twice, possibly with different spider arguments. I.e., this works:

``` python
crawler = Crawler(myspidercls)
crawler.crawl(my_sparg='blah')
crawler.crawl(my_sparg='other_blah')
```

This is done in some of our tests ([1](https://github.com/scrapy/scrapy/blob/master/tests/test_crawl.py#L63), [2](https://github.com/scrapy/scrapy/blob/master/tests/test_crawl.py#L132), [3](https://github.com/scrapy/scrapy/blob/master/tests/test_downloader_handlers.py#L299)). Are these tests legacy code or is support for multiple crawls part of the public API (see also [this discussion](https://github.com/scrapy/scrapy/pull/1580#issuecomment-154209093))?



Discussion:
@jdemaeyer pretty sure is legacy code. We should remove those tests to avoid confusion.

I think we can deprecate running crawl() for the second time after the first one finishes (they can't be run concurrently already).

As for tests I'm sure it's just a shortcut, not something written explicitly.

Resolving patch:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -124,6 +124,7 @@ def __init__(
 
         self.settings.freeze()
         self.crawling: bool = False
+        self._started: bool = False
         self.spider: Optional[Spider] = None
         self.engine: Optional[ExecutionEngine] = None
 
@@ -131,7 +132,13 @@ def __init__(
     def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
         if self.crawling:
             raise RuntimeError("Crawling already taking place")
-        self.crawling = True
+        if self._started:
+            warnings.warn(
+                "Running Crawler.crawl() more than once is deprecated.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        self.crawling = self._started = True
 
         try:
             self.spider = self._create_spider(*args, **kwargs)


Test patch:
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -123,7 +123,9 @@ def test_timeout_failure(self):
         self.assertTrue(crawler.spider.t2 == 0)
         self.assertTrue(crawler.spider.t2_err > 0)
         self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
+
         # server hangs after receiving response headers
+        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
         yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -201,6 +203,7 @@ def test_start_requests_dupes(self):
         )
         self.assertEqual(crawler.spider.visited, 6)
 
+        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
         yield crawler.crawl(
             dont_filter=False,
             distinct_urls=3,
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -6,6 +6,7 @@
 import warnings
 from pathlib import Path
 
+import pytest
 from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted.internet import defer
@@ -67,6 +68,16 @@ def test_crawler_rejects_spider_objects(self):
         with raises(ValueError):
             Crawler(DefaultSpider())
 
+    @defer.inlineCallbacks
+    def test_crawler_crawl_twice_deprecated(self):
+        crawler = Crawler(NoRequestsSpider)
+        yield crawler.crawl()
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"Running Crawler.crawl\(\) more than once is deprecated",
+        ):
+            yield crawler.crawl()
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -743,6 +743,7 @@ def test_download_gzip_response(self):
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
         raise unittest.SkipTest("xpayload fails on PY3")
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate")
         request = request.replace(url=self.mockserver.url("/xpayload"))
         yield crawler.crawl(seed=request)