Repository: scrapy/scrapy
Pull request #6911 (2025-06-25 12:35:53)

Narrow down TestEngine::test_short_timeout() expectations.
Fixes #6908

As it doesn't look like we can avoid the unhandled CancelledError, change the expectations to exceptions that could be raised when `self._slot` is unexpectedly None.

The test was added in #5440

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/scrapy/scrapy/pull/6911?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 84.89%. Comparing base [(`ef01a95`)](https://app.codecov.io/gh/scrapy/scrapy/commit/ef01a953b108b6c60d5ed5dcf85d3a1eb7ae7a2a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) to head [(`a4881f6`)](https://app.codecov.io/gh/scrapy/scrapy/commit/a4881f6380e447d3362c3e6644cc0b72ab519449?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy).

:white_check_mark: All tests successful. No failed tests found.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             2.13    #6911      +/-   ##
==========================================
- Coverage   84.89%   84.89%   -0.01%     
==========================================
  Files         164      164              
  Lines       12373    12365       -8     
  Branches     1610     1610              
==========================================
- Hits        10504    10497       -7     
+ Misses       1573     1561      -12     
- Partials      296      307      +11     
```

[see 15 files with indirect coverage changes](https://app.codecov.io/gh/scrapy/scrapy/pull/6911/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy)

</details>
wRAR: Ok, this is again stopping and starting in parallel, we may need more checks and/or some locking.

Patch:
diff --git a/docs/news.rst b/docs/news.rst
index b2fe78bc6..5fad2e425 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1219,6 +1219,17 @@ Security bug fixes
 
     .. _defusedxml: https://github.com/tiran/defusedxml
 
+Deprecations
+~~~~~~~~~~~~
+
+-   ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` (a spider
+    middleware) is now deprecated and not enabled by default. The new
+    downloader middleware with the same functionality,
+    :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, is enabled
+    instead.
+    (:issue:`2241`, :issue:`6358`)
+
+
 Bug fixes
 ~~~~~~~~~
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0f81a0c0a..bc2dd5286 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -503,7 +503,7 @@ performed by the Scrapy downloader.
 CONCURRENT_REQUESTS_PER_DOMAIN
 ------------------------------
 
-Default: ``8``
+Default: ``1`` (:ref:`fallback <default-settings>`: ``8``)
 
 The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single domain.
@@ -855,7 +855,7 @@ Whether to enable downloader stats collection.
 DOWNLOAD_DELAY
 --------------
 
-Default: ``0``
+Default: ``1`` (:ref:`fallback <default-settings>`: ``0``)
 
 Minimum seconds to wait between 2 consecutive requests to the same domain.
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b3a5e1906..1928f6d64 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -135,6 +135,9 @@ class ExecutionEngine:
         await maybe_deferred_to_future(
             self.signals.send_catch_log_deferred(signal=signals.engine_started)
         )
+        if _start_request_processing and self.spider is None:
+            # require an opened spider when not run in scrapy shell
+            return
         self.running = True
         self._closewait = Deferred()
         if _start_request_processing:
@@ -541,5 +544,6 @@ class ExecutionEngine:
         dfd.addErrback(log_failure("Error while unassigning spider"))
 
         dfd.addBoth(lambda _: self._spider_closed_callback(spider))
+        dfd.addErrback(log_failure("Error running spider_closed_callback"))
 
         return dfd
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index db7400af8..0432a7231 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -21,16 +21,10 @@ ADDONS = {}
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = True
 
-# Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
-
-# Configure a delay for requests for the same website (default: 0)
-# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
-# See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
-# The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
+# Concurrency and throttling settings
+#CONCURRENT_REQUESTS = 16
+CONCURRENT_REQUESTS_PER_DOMAIN = 1
+DOWNLOAD_DELAY = 1
 
 # Disable cookies (enabled by default)
 #COOKIES_ENABLED = False


Test patch:
diff --git a/tests/spiders.py b/tests/spiders.py
index c47f2bd2b..43cdcaef3 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -418,7 +418,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
     async def start(self):
         test_body = b"""
         <html>
-            <head><title>Page title<title></head>
+            <head><title>Page title</title></head>
             <body>
                 <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
                 <p><a href="/status?n=201">Item 201</a></p>  <!-- callback -->
@@ -472,7 +472,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
     async def start(self):
         test_body = b"""
         <html>
-            <head><title>Page title<title></head>
+            <head><title>Page title</title></head>
             <body>
                 <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
                 <p><a href="/status?n=201">Item 201</a></p>  <!-- callback -->
diff --git a/tests/test_engine.py b/tests/test_engine.py
index a3c5243d5..aa9c1b23d 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -509,7 +509,9 @@ class TestEngine(TestEngineBase):
         finally:
             timer.cancel()
 
-        assert b"Traceback" not in stderr, stderr
+        stderr_str = stderr.decode("utf-8")
+        assert "AttributeError" not in stderr_str, stderr_str
+        assert "AssertionError" not in stderr_str, stderr_str
 
 
 def test_request_scheduled_signal(caplog):
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 8211667fb..fd70de3d8 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -112,7 +112,8 @@ class MainTestCase(TestCase):
         with LogCapture(level=ERROR) as log:
             await maybe_deferred_to_future(crawler.crawl())
 
-        assert not log.records
+        assert len(log.records) == 1
+        assert log.records[0].msg == "Error running spider_closed_callback"
         finish_reason = crawler.stats.get_value("finish_reason")
         assert finish_reason == "shutdown", f"{finish_reason=}"
         expected_urls = []
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 1bff369af..15d358d2a 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -152,7 +152,7 @@ class Base:
         def test_nofollow(self):
             """Test the extractor's behaviour for links with rel='nofollow'"""
 
-            html = b"""<html><head><title>Page title<title>
+            html = b"""<html><head><title>Page title</title></head>
             <body>
             <div class='links'>
             <p><a href="/about.html">About us</a></p>
@@ -232,7 +232,7 @@ class Base:
 
         def test_restrict_xpaths_encoding(self):
             """Test restrict_xpaths with encodings"""
-            html = b"""<html><head><title>Page title<title>
+            html = b"""<html><head><title>Page title</title></head>
             <body><p><a href="item/12.html">Item 12</a></p>
             <div class='links'>
             <p><a href="/about.html">About us\xa3</a></p>
@@ -378,7 +378,7 @@ class Base:
             ]
 
         def test_base_url_with_restrict_xpaths(self):
-            html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
+            html = b"""<html><head><title>Page title</title><base href="http://otherdomain.com/base/" /></head>
             <body><p><a href="item/12.html">Item 12</a></p>
             </body></html>"""
             response = HtmlResponse("http://example.org/somepage/index.html", body=html)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index b4aa649a3..1f1e6c2b6 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -234,7 +234,7 @@ class TestCSVFeedSpider(TestSpider):
 
 
 class TestCrawlSpider(TestSpider):
-    test_body = b"""<html><head><title>Page title<title>
+    test_body = b"""<html><head><title>Page title</title></head>
     <body>
     <p><a href="item/12.html">Item 12</a></p>
     <div class='links'>