Repository: scrapy/scrapy
Pull request #6372 (2024-05-20 18:23:23)

Move CI from the decommissioned macos-11 to macos-latest.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/scrapy/scrapy/pull/6372?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 84.92%. Comparing base [(`180bc9b`)](https://app.codecov.io/gh/scrapy/scrapy/commit/180bc9bad7aceb3a9e10c1411212914bc32fb721?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) to head [(`18c9a14`)](https://app.codecov.io/gh/scrapy/scrapy/pull/6372?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy).
> Report is 13 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #6372      +/-   ##
==========================================
- Coverage   85.00%   84.92%   -0.08%     
==========================================
  Files         161      162       +1     
  Lines       11962    12112     +150     
  Branches     1872     1839      -33     
==========================================
+ Hits        10168    10286     +118     
- Misses       1512     1538      +26     
- Partials      282      288       +6     
```

[see 9 files with indirect coverage changes](https://app.codecov.io/gh/scrapy/scrapy/pull/6372/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy)

</details>
wRAR: https://github.com/wRAR/scrapy/actions/runs/9163227979

(3.8 was failing before too)

Patch:
diff --git a/.flake8 b/.flake8
index 62ccad9cf..cf1a96476 100644
--- a/.flake8
+++ b/.flake8
@@ -16,6 +16,7 @@ per-file-ignores =
     scrapy/linkextractors/__init__.py:E402,F401
     scrapy/selector/__init__.py:F401
     scrapy/spiders/__init__.py:E402,F401
+    tests/CrawlerRunner/change_reactor.py:E402
 
     # Issues pending a review:
     scrapy/utils/url.py:F403,F405
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index cd359b147..1500011e7 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running.
 
 .. code-block:: python
 
-    from twisted.internet import reactor
     import scrapy
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
@@ -107,6 +106,37 @@ reactor after ``MySpider`` has finished running.
     runner = CrawlerRunner()
 
     d = runner.crawl(MySpider)
+
+    from twisted.internet import reactor
+
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run()  # the script will block here until the crawling is finished
+
+Same example but using a non-default reactor, it's only necessary call
+``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
+
+.. code-block:: python
+
+    import scrapy
+    from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
+
+
+    class MySpider(scrapy.Spider):
+        # Your spider definition
+        ...
+
+
+    configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
+
+    from scrapy.utils.reactor import install_reactor
+
+    install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+    runner = CrawlerRunner()
+    d = runner.crawl(MySpider)
+
+    from twisted.internet import reactor
+
     d.addBoth(lambda _: reactor.stop())
     reactor.run()  # the script will block here until the crawling is finished
 
@@ -151,7 +181,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
 .. code-block:: python
 
     import scrapy
-    from twisted.internet import reactor
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
@@ -173,6 +202,9 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
     d = runner.join()
+
+    from twisted.internet import reactor
+
     d.addBoth(lambda _: reactor.stop())
 
     reactor.run()  # the script will block here until all crawling jobs are finished
@@ -181,7 +213,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
 
 .. code-block:: python
 
-    from twisted.internet import reactor, defer
+    from twisted.internet import defer
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
@@ -209,6 +241,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
         reactor.stop()
 
 
+    from twisted.internet import reactor
+
     crawl()
     reactor.run()  # the script will block here until the last crawl call is finished
 
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 9a0811a50..1a3d36f45 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -59,7 +59,8 @@ class S3DownloadHandler:
             assert aws_access_key_id is not None
             assert aws_secret_access_key is not None
             SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
-            self._signer = SignerCls(
+            # botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do
+            self._signer = SignerCls(  # type: ignore[call-arg]
                 botocore.credentials.Credentials(
                     aws_access_key_id, aws_secret_access_key, aws_session_token
                 )
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ccfe78891..4fe5987a7 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -129,6 +129,8 @@ class Crawler:
             if is_asyncio_reactor_installed() and event_loop:
                 verify_installed_asyncio_event_loop(event_loop)
 
+            log_reactor_info()
+
         self.extensions = ExtensionManager.from_crawler(self)
         self.settings.freeze()
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 97f39afe7..3c2bb5593 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -238,13 +238,16 @@ class S3FeedStorage(BlockingFeedStorage):
         self.acl: Optional[str] = acl
         self.endpoint_url: Optional[str] = endpoint_url
         self.region_name: Optional[str] = region_name
+        # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,
+        # there seems to be no good way to infer it statically.
+        self.s3_client: Any
 
         if IS_BOTO3_AVAILABLE:
             import boto3.session
 
-            session = boto3.session.Session()
+            boto3_session = boto3.session.Session()
 
-            self.s3_client = session.client(
+            self.s3_client = boto3_session.client(
                 "s3",
                 aws_access_key_id=self.access_key,
                 aws_secret_access_key=self.secret_key,
@@ -261,9 +264,9 @@ class S3FeedStorage(BlockingFeedStorage):
 
             import botocore.session
 
-            session = botocore.session.get_session()
+            botocore_session = botocore.session.get_session()
 
-            self.s3_client = session.create_client(
+            self.s3_client = botocore_session.create_client(
                 "s3",
                 aws_access_key_id=self.access_key,
                 aws_secret_access_key=self.secret_key,
diff --git a/tox.ini b/tox.ini
index cde4243f3..5a5e80496 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,12 +48,15 @@ basepython = python3
 deps =
     mypy==1.10.0
     typing-extensions==4.11.0
-    types-attrs==19.1.0
     types-lxml==2024.4.14
-    types-Pillow==10.2.0.20240423
-    types-Pygments==2.17.0.20240310
-    types-pyOpenSSL==24.0.0.20240417
-    types-setuptools==69.5.0.20240423
+    types-Pygments==2.18.0.20240506
+    types-pyOpenSSL==24.1.0.20240425
+    types-setuptools==69.5.0.20240518
+    botocore-stubs==1.34.94
+    boto3-stubs[s3]==1.34.108
+    attrs >= 18.2.0
+    Pillow >= 10.3.0
+    pytest >= 8.2.0
     # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
     w3lib >= 2.1.2
 commands =


Test patch:
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 252176464..a297f494c 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ concurrency:
 
 jobs:
   tests:
-    runs-on: macos-11
+    runs-on: macos-latest
     strategy:
       fail-fast: false
       matrix:
diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py
new file mode 100644
index 000000000..b20aa0c7c
--- /dev/null
+++ b/tests/CrawlerRunner/change_reactor.py
@@ -0,0 +1,31 @@
+from scrapy import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.log import configure_logging
+
+
+class NoRequestsSpider(Spider):
+    name = "no_request"
+
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+    def start_requests(self):
+        return []
+
+
+configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"})
+
+
+from scrapy.utils.reactor import install_reactor
+
+install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+runner = CrawlerRunner()
+
+d = runner.crawl(NoRequestsSpider)
+
+from twisted.internet import reactor
+
+d.addBoth(callback=lambda _: reactor.stop())
+reactor.run()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 989208694..791ea1faa 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("INFO: Host: not.a.real.domain", log)
         self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
         self.assertIn("INFO: IP address: 127.0.0.1", log)
+
+    def test_change_default_reactor(self):
+        log = self.run_script("change_reactor.py")
+        self.assertIn(
+            "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            log,
+        )
+        self.assertIn("DEBUG: Using asyncio event loop", log)