Repository: scrapy/scrapy

Issue:
scrapy engine can stop even if start_requests is not empty
Consider this example spider

``` python
from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy.http import Request

class DmozSpider(Spider):
    name = "dmoz"
    allowed_domains = ["dmoz.org"]
    start_urls = [
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
    ]

    def start_requests(self):
        for url in self.start_urls:
            yield Request(
                url,
                #dont_filter=True
            )

    def parse(self, response):
        self.log("parse %r" % response.url)

```

and run it with `CONCURRENT_REQUESTS=1` and `DupeFilter` enabled,
it will only visit `http://www.dmoz.org/Computers/Programming/Languages/Python/Books/`

Even with higher concurrency settings, it can happen that the spider is considered idle because the next request from `start_requests` iterator was filtered.

https://github.com/scrapy/scrapy/blob/master/scrapy/core/engine.py#L120 or https://github.com/scrapy/scrapy/blob/master/scrapy/core/engine.py#L155 seems to be missing a test on `slot.start_requests is None`



Discussion:

Resolving patch:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -112,6 +112,7 @@ def _next_request(self, spider):
             except StopIteration:
                 slot.start_requests = None
             except Exception as exc:
+                slot.start_requests = None
                 log.err(None, 'Obtaining request from start requests', \
                         spider=spider)
             else:
@@ -156,7 +157,8 @@ def spider_is_idle(self, spider):
         scraper_idle = self.scraper.slot.is_idle()
         pending = self.slot.scheduler.has_pending_requests()
         downloading = bool(self.downloader.active)
-        idle = scraper_idle and not (pending or downloading)
+        pending_start_requests = self.slot.start_requests is not None
+        idle = scraper_idle and not (pending or downloading or pending_start_requests)
         return idle
 
     @property


Test patch:
diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py
--- a/scrapy/tests/spiders.py
+++ b/scrapy/tests/spiders.py
@@ -157,3 +157,26 @@ def on_error(self, failure):
         self.meta['failure'] = failure
         if callable(self.errback_func):
             return self.errback_func(failure)
+
+
+class DuplicateStartRequestsSpider(Spider):
+    dont_filter = True
+    name = 'duplicatestartrequests'
+    distinct_urls = 2
+    dupe_factor = 3
+
+    def start_requests(self):
+        for i in range(0, self.distinct_urls):
+            for j in range(0, self.dupe_factor):
+                url = "http://localhost:8998/echo?headers=1&body=test%d" % i
+                yield self.make_requests_from_url(url)
+
+    def make_requests_from_url(self, url):
+        return Request(url, dont_filter=self.dont_filter)
+
+    def __init__(self, url="http://localhost:8998", *args, **kwargs):
+        super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
+        self.visited = 0
+
+    def parse(self, response):
+        self.visited += 1
diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py
--- a/scrapy/tests/test_crawl.py
+++ b/scrapy/tests/test_crawl.py
@@ -5,7 +5,7 @@
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import docrawl, get_testlog
 from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
-    BrokenStartRequestsSpider, SingleRequestSpider
+    BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from scrapy.tests.mockserver import MockServer
 from scrapy.http import Request
 
@@ -113,6 +113,21 @@ def test_start_requests_lazyness(self):
         #self.assertTrue(spider.seedsseen.index(None) < spider.seedsseen.index(99),
         #                spider.seedsseen)
 
+    @defer.inlineCallbacks
+    def test_start_requests_dupes(self):
+        settings = {"CONCURRENT_REQUESTS": 1}
+        spider = DuplicateStartRequestsSpider(dont_filter=True,
+                                              distinct_urls=2,
+                                              dupe_factor=3)
+        yield docrawl(spider, settings)
+        self.assertEqual(spider.visited, 6)
+
+        spider = DuplicateStartRequestsSpider(dont_filter=False,
+                                              distinct_urls=3,
+                                              dupe_factor=4)
+        yield docrawl(spider, settings)
+        self.assertEqual(spider.visited, 3)
+
     @defer.inlineCallbacks
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding