Issue 1:
Regarding the spider_idle signal, the doc says:

If the idle state persists after all handlers of this signal have finished, the engine starts closing the spider. After the spider has finished closing, the spider_closed signal is sent.

You can, for example, schedule some requests in your spider_idle handler to prevent the spider from being closed.

However, the spider may still close even when it has scheduled some requests within a spider_idle handler, if all such requests are filtered due to duplication.

In this case, the engine would still have self.spider_is_idle(spider) evaluated to True after finishing all spider_idle handlers.

Related code in scrapy/core/engine.py

Below is an example illustrating this issue. In this example, one may expect the spider to send three requests, but with the current version of Scrapy (05ce1296) there'll be only one request.

# coding: utf8

import scrapy
import scrapy.signals

class TestSpider(scrapy.Spider):

name = 'test'
urls = None

def start_requests(self):
self.urls = [
'http://httpbin.org/get?foo=bar',
'http://httpbin.org/get?bar=baz',
'http://httpbin.org/get?baz=foo',
'http://httpbin.org/get?baz=foo',
]
self.crawler.signals.connect(self.on_idle, scrapy.signals.spider_idle)
return []

def on_idle(self, spider):
if self.urls:
url = self.urls.pop()
self.crawler.engine.crawl(scrapy.Request(url), self)

def parse(self, response):
pass

(Save it to a.py and use scrapy runspider a.py to test.)

My suggestion is to have ExecutionEngine._spider_idle updated to make sure the spider does not get closed, as long as there's any request scheduled within a spider_idle handler.

Issue 2:
It seems to be failing in CI environment excepts where dependency versions are pinned. I suspect it may be because of the latest w3lib release.

_____________ CrawlSpiderTest.test_process_request_instance_method _____________

self = <tests.test_spider.CrawlSpiderTest testMethod=test_process_request_instance_method>

def test_process_request_instance_method(self):

response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)

class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_request='process_request_upper'),
)

def process_request_upper(self, request, response):
return request.replace(url=request.url.upper())

spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
> self.assertEqual([r.url for r in output],
['http://example.org/SOMEPAGE/ITEM/12.HTML',
'http://example.org/ABOUT.HTML',
'http://example.org/NOFOLLOW.HTML'])

/home/runner/work/scrapy/scrapy/tests/test_spider.py:363:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/home/runner/work/scrapy/scrapy/.tox/py/lib/python3.8/site-packages/twisted/trial/_synctest.py:422: in assertEqual
super().assertEqual(first, second, msg)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <tests.test_spider.CrawlSpiderTest testMethod=test_process_request_instance_method>
msg = "Lists differ: ['http://example.org/SOMEPAGE/ITEM/12.HTML', 'http[58 chars]TML'] != ['http://example.org/SOMEPAGE/ITEM...,\n- 'http://example.org/NOFOLLOW.HTML']\n+ 'http://example.org/ABOUT.HTML',\n+ 'http://example.org/NOFOLLOW.HTML']"

def fail(self, msg=None):
"""
Absolutely fail the test. Do not pass go, do not collect $200.

@param msg: the message that will be displayed as the reason for the
failure
"""
> raise self.failureException(msg)
E twisted.trial.unittest.FailTest: Lists differ: ['http://example.org/SOMEPAGE/ITEM/12.HTML', 'http[58 chars]TML'] != ['http://example.org/SOMEPAGE/ITEM/12.HTML', 'http[58 chars]TML']
E
E First differing element 0:
E 'http://example.org/SOMEPAGE/ITEM/12.HTML'
E 'http://example.org/SOMEPAGE/ITEM/12.HTML'
E
E - ['http://example.org/SOMEPAGE/ITEM/12.HTML',
E ? ^^^^^^^ ^^^
E
E + ['http://example.org/SOMEPAGE/ITEM/12.HTML',
E ? ^^^^^^^ ^^^
E
E - 'http://example.org/ABOUT.HTML',
E - 'http://example.org/NOFOLLOW.HTML']
E + 'http://example.org/ABOUT.HTML',
E + 'http://example.org/NOFOLLOW.HTML']

There is a second issue about IPv6 resolving, but I wonder if it could also be related to w3lib:

___________ CrawlerProcessSubprocess.test_ipv6_default_name_resolver ___________

self = <tests.test_crawler.CrawlerProcessSubprocess testMethod=test_ipv6_default_name_resolver>

def test_ipv6_default_name_resolver(self):
log = self.run_script('default_name_resolver.py')
self.assertIn('Spider closed (finished)', log)
> self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)

/home/runner/work/scrapy/scrapy/tests/test_crawler.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <tests.test_crawler.CrawlerProcessSubprocess testMethod=test_ipv6_default_name_resolver>
containee = "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,"
container = '/home/runner/work/scrapy/scrapy/.tox/py/lib/python3.8/site-packages/coverage/inorout.py:472: CoverageWarning: --inclu...me.datetime(2022, 8, 28, 16, 9, 4, 748022)}\n2022-08-28 16:09:04 [scrapy.core.engine] INFO: Spider closed (finished)\n'
msg = None

def assertIn(self, containee, container, msg=None):
"""
Fail the test if C{containee} is not found in C{container}.

@param containee: the value that should be in C{container}
@param container: a sequence type, or in the case of a mapping type,
will follow semantics of 'if key in dict.keys()'
@param msg: if msg is None, then the failure message will be
'%r not in %r' % (first, second)
"""
if containee not in container:
> raise self.failureException(msg or f"{containee!r} not in {container!r}")
E twisted.trial.unittest.FailTest: "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1," not in '/home/runner/work/scrapy/scrapy/.tox/py/lib/python3.8/site-packages/coverage/inorout.py:472: CoverageWarning: --include is ignored because --source is set (include-ignored)\n self.warn("--include is ignored because --source is set", slug="include-ignored")\n2022-08-28 16:09:04 [scrapy.utils.log] INFO: Scrapy 2.6.2 started (bot: scrapybot)\n2022-08-28 16:09:04 [scrapy.utils.log] INFO: Versions: lxml 4.9.1.0, libxml2 2.9.14, cssselect 1.1.0, parsel 1.6.0, w3lib 2.0.1, Twisted 22.4.0, Python 3.8.13 (default, Jul 22 2022, 10:42:37) - [GCC 7.5.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1l 24 Aug 2021), cryptography 3.4.8, Platform Linux-5.4.0-1086-azure-x86_64-with-glibc2.2.5\n2022-08-28 16:09:04 [scrapy.crawler] INFO: Overridden settings:\n{\'RETRY_ENABLED\': False}\n2022-08-28 16:09:04 [py.warnings] WARNING: /home/runner/work/scrapy/scrapy/scrapy/utils/request.py:[231](https://github.com/scrapy/scrapy/runs/8058868863?check_suite_focus=true#step:5:232): ScrapyDeprecationWarning: \'PREVIOUS_VERSION\' is a deprecated value for the \'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the \'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the \'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2022-08-28 16:09:04 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor\n2022-08-28 16:09:04 [scrapy.extensions.telnet] INFO: Telnet Password: 753efe10088dde0a\n2022-08-28 16:09:04 [scrapy.middleware] INFO: Enabled extensions:\n[\'scrapy.extensions.corestats.CoreStats\',\n \'scrapy.extensions.telnet.TelnetConsole\',\n \'scrapy.extensions.memusage.MemoryUsage\',\n \'scrapy.extensions.logstats.LogStats\']\n2022-08-28 16:09:04 [scrapy.middleware] INFO: Enabled downloader middlewares:\n[\'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware\',\n \'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware\',\n \'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware\',\n \'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware\',\n \'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware\',\n \'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware\',\n \'scrapy.downloadermiddlewares.redirect.RedirectMiddleware\',\n \'scrapy.downloadermiddlewares.cookies.CookiesMiddleware\',\n \'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware\',\n \'scrapy.downloadermiddlewares.stats.DownloaderStats\']\n2022-08-28 16:09:04 [scrapy.middleware] INFO: Enabled spider middlewares:\n[\'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware\',\n \'scrapy.spidermiddlewares.offsite.OffsiteMiddleware\',\n \'scrapy.spidermiddlewares.referer.RefererMiddleware\',\n \'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware\',\n \'scrapy.spidermiddlewares.depth.DepthMiddleware\']\n2022-08-28 16:09:04 [scrapy.middleware] INFO: Enabled item pipelines:\n[]\n2022-08-28 16:09:04 [scrapy.core.engine] INFO: Spider opened\n2022-08-28 16:09:04 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2022-08-28 16:09:04 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n2022-08-28 16:09:04 [scrapy.core.scraper] ERROR: Error downloading <GET http://::1>\nTraceback (most recent call last):\n File "/home/runner/work/scrapy/scrapy/scrapy/core/downloader/middleware.py", line 49, in process_request\n return (yield download_func(request=request, spider=spider))\nValueError: invalid hostname: :\n2022-08-28 16:09:04 [scrapy.core.engine] INFO: Closing spider (finished)\n2022-08-28 16:09:04 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{\'downloader/exception_count\': 1,\n \'downloader/exception_type_count/builtins.ValueError\': 1,\n \'downloader/request_bytes\': 210,\n \'downloader/request_count\': 1,\n \'downloader/request_method_count/GET\': 1,\n \'elapsed_time_seconds\': 0.208906,\n \'finish_reason\': \'finished\',\n \'finish_time\': datetime.datetime(2022, 8, 28, 16, 9, 4, 956928),\n \'log_count/DEBUG\': 1,\n \'log_count/ERROR\': 1,\n \'log_count/INFO\': 10,\n \'log_count/WARNING\': 1,\n \'memusage/max\': 144793600,\n \'memusage/startup\': 144793600,\n \'scheduler/dequeued\': 1,\n \'scheduler/dequeued/memory\': 1,\n \'scheduler/enqueued\': 1,\n \'scheduler/enqueued/memory\': 1,\n \'start_time\': datetime.datetime(2022, 8, 28, 16, 9, 4, 748022)}\n2022-08-28 16:09:04 [scrapy.core.engine] INFO: Spider closed (finished)\n'