Repository: scrapy/scrapy (branch: master)
Commit date: 2023-10-18T02:00:37Z

Workflow file: .github/workflows/checks.yml (Checks)
name: Checks
on: [push, pull_request]

concurrency:
  group: ${{github.workflow}}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  checks:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    strategy:
      fail-fast: false
      matrix:
        include:
        - python-version: "3.12"
          env:
            TOXENV: pylint
        - python-version: 3.8
          env:
            TOXENV: typing
        - python-version: "3.11"  # Keep in sync with .readthedocs.yml
          env:
            TOXENV: docs
        - python-version: "3.12"
          env:
            TOXENV: twinecheck

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Run check
      env: ${{ matrix.env }}
      run: |
        pip install -U tox
        tox

  pre-commit:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    steps:
    - uses: actions/checkout@v4
    - uses: pre-commit/action@v3.0.0

env:
  PIP_INDEX_URL: http://localhost:8629/2023-10-18
  UV_INDEX_URL: http://localhost:8629/2023-10-18


Failed build logs:
step_name: checks (3.8, typing)/4_Run check.txt
log: ##[group]Run pip install -U tox
[36;1mpip install -U tox[0m
[36;1mtox[0m
shell: /usr/bin/bash -e {0}
env:
  pythonLocation: /opt/hostedtoolcache/Python/3.8.18/x64
  PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.8.18/x64/lib/pkgconfig
  Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.8.18/x64
  Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.8.18/x64
  Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.8.18/x64
  LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.18/x64/lib
  TOXENV: typing
##[endgroup]
Collecting tox
  Obtaining dependency information for tox from https://files.pythonhosted.org/packages/f5/f9/963052e8b825645c54262dce7b7c88691505e3b9ee10a3e3667711eaaf21/tox-4.11.3-py3-none-any.whl.metadata
  Downloading tox-4.11.3-py3-none-any.whl.metadata (5.0 kB)
Collecting cachetools>=5.3.1 (from tox)
  Obtaining dependency information for cachetools>=5.3.1 from https://files.pythonhosted.org/packages/a9/c9/c8a7710f2cedcb1db9224fdd4d8307c9e48cbddc46c18b515fefc0f1abbe/cachetools-5.3.1-py3-none-any.whl.metadata
  Downloading cachetools-5.3.1-py3-none-any.whl.metadata (5.2 kB)
Collecting chardet>=5.2 (from tox)
  Obtaining dependency information for chardet>=5.2 from https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl.metadata
  Downloading chardet-5.2.0-py3-none-any.whl.metadata (3.4 kB)
Collecting colorama>=0.4.6 (from tox)
  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)
Collecting filelock>=3.12.3 (from tox)
  Obtaining dependency information for filelock>=3.12.3 from https://files.pythonhosted.org/packages/5e/5d/97afbafd9d584ff1b45fcb354a479a3609bd97f912f8f1f6c563cb1fae21/filelock-3.12.4-py3-none-any.whl.metadata
  Downloading filelock-3.12.4-py3-none-any.whl.metadata (2.8 kB)
Collecting packaging>=23.1 (from tox)
  Obtaining dependency information for packaging>=23.1 from https://files.pythonhosted.org/packages/ec/1a/610693ac4ee14fcdf2d9bf3c493370e4f2ef7ae2e19217d7a237ff42367d/packaging-23.2-py3-none-any.whl.metadata
  Downloading packaging-23.2-py3-none-any.whl.metadata (3.2 kB)
Collecting platformdirs>=3.10 (from tox)
  Obtaining dependency information for platformdirs>=3.10 from https://files.pythonhosted.org/packages/56/29/3ec311dc18804409ecf0d2b09caa976f3ae6215559306b5b530004e11156/platformdirs-3.11.0-py3-none-any.whl.metadata
  Downloading platformdirs-3.11.0-py3-none-any.whl.metadata (11 kB)
Collecting pluggy>=1.3 (from tox)
  Obtaining dependency information for pluggy>=1.3 from https://files.pythonhosted.org/packages/05/b8/42ed91898d4784546c5f06c60506400548db3f7a4b3fb441cba4e5c17952/pluggy-1.3.0-py3-none-any.whl.metadata
  Downloading pluggy-1.3.0-py3-none-any.whl.metadata (4.3 kB)
Collecting pyproject-api>=1.6.1 (from tox)
  Obtaining dependency information for pyproject-api>=1.6.1 from https://files.pythonhosted.org/packages/cf/b4/39eea50542e50e93876ebc09c4349a9c9eee9f6b9c9d30f88c7dc5433db8/pyproject_api-1.6.1-py3-none-any.whl.metadata
  Downloading pyproject_api-1.6.1-py3-none-any.whl.metadata (2.8 kB)
Collecting tomli>=2.0.1 (from tox)
  Downloading tomli-2.0.1-py3-none-any.whl (12 kB)
Collecting virtualenv>=20.24.3 (from tox)
  Obtaining dependency information for virtualenv>=20.24.3 from https://files.pythonhosted.org/packages/4e/8b/f0d3a468c0186c603217a6656ea4f49259630e8ed99558501d92f6ff7dc3/virtualenv-20.24.5-py3-none-any.whl.metadata
  Downloading virtualenv-20.24.5-py3-none-any.whl.metadata (4.5 kB)
Collecting distlib<1,>=0.3.7 (from virtualenv>=20.24.3->tox)
  Obtaining dependency information for distlib<1,>=0.3.7 from https://files.pythonhosted.org/packages/43/a0/9ba967fdbd55293bacfc1507f58e316f740a3b231fc00e3d86dc39bc185a/distlib-0.3.7-py2.py3-none-any.whl.metadata
  Downloading distlib-0.3.7-py2.py3-none-any.whl.metadata (5.1 kB)
Downloading tox-4.11.3-py3-none-any.whl (153 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 153.8/153.8 kB 19.6 MB/s eta 0:00:00
Downloading cachetools-5.3.1-py3-none-any.whl (9.3 kB)
Downloading chardet-5.2.0-py3-none-any.whl (199 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 199.4/199.4 kB 54.3 MB/s eta 0:00:00
Downloading filelock-3.12.4-py3-none-any.whl (11 kB)
Downloading packaging-23.2-py3-none-any.whl (53 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.0/53.0 kB 18.9 MB/s eta 0:00:00
Downloading platformdirs-3.11.0-py3-none-any.whl (17 kB)
Downloading pluggy-1.3.0-py3-none-any.whl (18 kB)
Downloading pyproject_api-1.6.1-py3-none-any.whl (12 kB)
Downloading virtualenv-20.24.5-py3-none-any.whl (3.7 MB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.7/3.7 MB 101.6 MB/s eta 0:00:00
Downloading distlib-0.3.7-py2.py3-none-any.whl (468 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 468.9/468.9 kB 83.7 MB/s eta 0:00:00
Installing collected packages: distlib, tomli, pluggy, platformdirs, packaging, filelock, colorama, chardet, cachetools, virtualenv, pyproject-api, tox
Successfully installed cachetools-5.3.1 chardet-5.2.0 colorama-0.4.6 distlib-0.3.7 filelock-3.12.4 packaging-23.2 platformdirs-3.11.0 pluggy-1.3.0 pyproject-api-1.6.1 tomli-2.0.1 tox-4.11.3 virtualenv-20.24.5

[notice] A new release of pip is available: 23.0.1 -> 23.3
[notice] To update, run: pip install --upgrade pip
typing: install_deps> python -I -m pip install -ctests/upper-constraints.txt mypy==1.5.1 types-attrs==19.1.0 types-lxml==2023.3.28 types-Pillow==10.0.0.3 types-Pygments==2.16.0.0 types-pyOpenSSL==23.2.0.2 types-setuptools==68.2.0.0 typing-extensions==4.7.1 'w3lib>=2.1.2'
.pkg: install_requires> python -I -m pip install 'setuptools>=40.8.0' wheel
.pkg: _optional_hooks> python /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: get_requires_for_build_sdist> python /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: get_requires_for_build_wheel> python /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: install_requires_for_build_wheel> python -I -m pip install wheel
.pkg: freeze> python -m pip freeze --all
.pkg: pip==23.2.1,setuptools==68.2.0,wheel==0.41.2
.pkg: prepare_metadata_for_build_wheel> python /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: build_sdist> python /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
typing: install_package_deps> python -I -m pip install -ctests/upper-constraints.txt 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"' 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"' 'Twisted>=18.9.0' 'cryptography>=36.0.0' 'cssselect>=0.9.1' 'itemadapter>=0.1.0' 'itemloaders>=1.0.1' 'lxml>=4.4.1' packaging 'parsel>=1.5.0' 'protego>=0.1.15' 'pyOpenSSL>=21.0.0' 'queuelib>=1.4.2' 'service-identity>=18.1.0' setuptools tldextract 'w3lib>=1.17.0' 'zope.interface>=5.1.0'
typing: install_package> python -I -m pip install -ctests/upper-constraints.txt --force-reinstall --no-deps /home/runner/work/scrapy/scrapy/.tox/.tmp/package/1/Scrapy-2.11.0.tar.gz
typing: freeze> python -m pip freeze --all
typing: attrs==23.1.0,Automat==22.10.0,certifi==2023.7.22,cffi==1.16.0,charset-normalizer==3.3.0,constantly==15.1.0,cryptography==41.0.4,cssselect==1.2.0,filelock==3.12.4,hyperlink==21.0.0,idna==3.4,incremental==22.10.0,itemadapter==0.8.0,itemloaders==1.1.0,jmespath==1.0.1,lxml==4.9.3,mypy==1.5.1,mypy-extensions==1.0.0,packaging==23.2,parsel==1.8.1,pip==23.3,Protego==0.3.0,pyasn1==0.5.0,pyasn1-modules==0.3.0,pycparser==2.21,PyDispatcher==2.0.7,pyOpenSSL==23.2.0,queuelib==1.6.2,requests==2.31.0,requests-file==1.5.1,Scrapy @ file:///home/runner/work/scrapy/scrapy/.tox/.tmp/package/1/Scrapy-2.11.0.tar.gz#sha256=4d445cc983c6a263ec00575a9fe7de7dc03f7e00eedb8629eb85077c420d42e7,service-identity==23.1.0,setuptools==68.2.2,six==1.16.0,tldextract==5.0.1,tomli==2.0.1,Twisted==23.8.0,types-attrs==19.1.0,types-beautifulsoup4==4.12.0.6,types-docutils==0.20.0.3,types-html5lib==1.1.11.15,types-lxml==2023.3.28,types-Pillow==10.0.0.3,types-Pygments==2.16.0.0,types-pyOpenSSL==23.2.0.2,types-setuptools==68.2.0.0,typing_extensions==4.7.1,urllib3==2.0.7,w3lib==2.1.2,wheel==0.41.2,zope.interface==6.1
typing: commands[0]> mypy scrapy tests
scrapy/utils/log.py:244: error: Incompatible return value type (got "Tuple[Any, Optional[Any], Any]", expected "Tuple[int, str, Dict[Any, Any]]")  [return-value]
Found 1 error in 1 file (checked 338 source files)
typing: exit 1 (10.57 seconds) /home/runner/work/scrapy/scrapy> mypy scrapy tests pid=1884
.pkg: _exit> python /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
  typing: FAIL code 1 (31.11=setup[20.53]+cmd[10.57] seconds)
  evaluation failed :( (31.27 seconds)
##[error]Process completed with exit code 1.

step_name: pre-commit/3_Run pre-commitaction@v3.0.0.txt
log: ##[group]Run pre-commit/action@v3.0.0
with:
  extra_args: --all-files
##[endgroup]
##[group]Run python -m pip install pre-commit
[36;1mpython -m pip install pre-commit[0m
shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
##[endgroup]
Defaulting to user installation because normal site-packages is not writeable
Collecting pre-commit
  Downloading pre_commit-3.5.0-py2.py3-none-any.whl (203 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 203.7/203.7 KB 3.7 MB/s eta 0:00:00
Collecting identify>=1.0.0
  Downloading identify-2.5.30-py2.py3-none-any.whl (98 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 98.9/98.9 KB 20.1 MB/s eta 0:00:00
Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from pre-commit) (5.4.1)
Collecting cfgv>=2.0.0
  Downloading cfgv-3.4.0-py2.py3-none-any.whl (7.2 kB)
Collecting virtualenv>=20.10.0
  Downloading virtualenv-20.24.5-py3-none-any.whl (3.7 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.7/3.7 MB 69.0 MB/s eta 0:00:00
Collecting nodeenv>=0.11.1
  Downloading nodeenv-1.8.0-py2.py3-none-any.whl (22 kB)
Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from nodeenv>=0.11.1->pre-commit) (59.6.0)
Collecting filelock<4,>=3.12.2
  Downloading filelock-3.12.4-py3-none-any.whl (11 kB)
Collecting distlib<1,>=0.3.7
  Downloading distlib-0.3.7-py2.py3-none-any.whl (468 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 468.9/468.9 KB 57.0 MB/s eta 0:00:00
Collecting platformdirs<4,>=3.9.1
  Downloading platformdirs-3.11.0-py3-none-any.whl (17 kB)
Installing collected packages: distlib, platformdirs, nodeenv, identify, filelock, cfgv, virtualenv, pre-commit
Successfully installed cfgv-3.4.0 distlib-0.3.7 filelock-3.12.4 identify-2.5.30 nodeenv-1.8.0 platformdirs-3.11.0 pre-commit-3.5.0 virtualenv-20.24.5
##[group]Run python -m pip freeze --local
[36;1mpython -m pip freeze --local[0m
shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
##[endgroup]
argcomplete==3.1.2
cfgv==3.4.0
distlib==0.3.7
filelock==3.12.4
identify==2.5.30
nodeenv==1.8.0
packaging==23.2
pipx==1.2.0
platformdirs==3.11.0
pre-commit==3.5.0
userpath==1.9.1
virtualenv==20.24.5
##[group]Run actions/cache@v3
with:
  path: ~/.cache/pre-commit
  key: pre-commit-3||7a8fe885594aed9a90fd5938b4bb49b65732538a44c08aad3d6ea69d9d0cf64c
  enableCrossOsArchive: false
  fail-on-cache-miss: false
  lookup-only: false
##[endgroup]
Cache Size: ~33 MB (34938818 B)
[command]/usr/bin/tar -xf /home/runner/work/_temp/2e73504e-c13e-4da4-97ff-45c557d92a27/cache.tzst -P -C /home/runner/work/scrapy/scrapy --use-compress-program unzstd
Received 34938818 of 34938818 (100.0%), 33.3 MBs/sec
Cache restored successfully
Cache restored from key: pre-commit-3||7a8fe885594aed9a90fd5938b4bb49b65732538a44c08aad3d6ea69d9d0cf64c
##[group]Run pre-commit run --show-diff-on-failure --color=always --all-files
[36;1mpre-commit run --show-diff-on-failure --color=always --all-files[0m
shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
##[endgroup]
bandit...................................................................[42mPassed[m
flake8...................................................................[41mFailed[m
[2m- hook id: flake8[m
[2m- exit code: 1[m

[1mscrapy/utils/response.py[m[36m:[m17[36m:[m1[36m:[m [1m[31mF401[m 'scrapy.utils.decorators.deprecated' imported but unused
[1mtests/test_downloadermiddleware_stats.py[m[36m:[m12[36m:[m1[36m:[m [1m[31mE302[m expected 2 blank lines, found 1
[1mtests/test_utils_response.py[m[36m:[m2[36m:[m1[36m:[m [1m[31mF401[m 'warnings' imported but unused
[1mtests/test_utils_response.py[m[36m:[m6[36m:[m1[36m:[m [1m[31mF401[m 'scrapy.exceptions.ScrapyDeprecationWarning' imported but unused

black....................................................................[41mFailed[m
[2m- hook id: black[m
[2m- files were modified by this hook[m

[1mreformatted tests/test_downloadermiddleware_stats.py[0m

[1mAll done! ✨ 🍰 ✨[0m
[34m[1m1 file [0m[1mreformatted[0m, [34m345 files [0mleft unchanged.

isort....................................................................[41mFailed[m
[2m- hook id: isort[m
[2m- files were modified by this hook[m

Fixing /home/runner/work/scrapy/scrapy/tests/test_downloadermiddleware_stats.py

blacken-docs.............................................................[42mPassed[m
pre-commit hook(s) made changes.
If you are seeing this message in CI, reproduce locally with: `pre-commit run --all-files`.
To run `pre-commit` as part of git workflow, use `pre-commit install`.
All changes made by hooks:
[1mdiff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py[m
[1mindex 766b965..eda5a0a 100644[m
[1m--- a/tests/test_downloadermiddleware_stats.py[m
[1m+++ b/tests/test_downloadermiddleware_stats.py[m
[36m@@ -6,8 +6,9 @@[m [mfrom scrapy.downloadermiddlewares.stats import DownloaderStats[m
 from scrapy.exceptions import ScrapyDeprecationWarning[m
 from scrapy.http import Request, Response[m
 from scrapy.spiders import Spider[m
[31m-from scrapy.utils.test import get_crawler[m
 from scrapy.utils.python import to_bytes[m
[32m+[m[32mfrom scrapy.utils.test import get_crawler[m
[32m+[m
 [m
 class MyException(Exception):[m
     pass[m
##[error]Process completed with exit code 1.


Changed files:
docs/topics/downloader-middleware.rst
scrapy/robotstxt.py
scrapy/utils/log.py
scrapy/utils/response.py
tests/test_downloadermiddleware_stats.py
tests/test_utils_response.py

Fix diff:
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a8e5b23bf..1abbc4968 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1039,8 +1039,8 @@ RobotsTxtMiddleware
 
     * :ref:`Protego <protego-parser>` (default)
     * :ref:`RobotFileParser <python-robotfileparser>`
-    * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
+    * :ref:`Reppy <reppy-parser>` (deprecated)
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1133,6 +1133,7 @@ In order to use this parser:
 
     .. warning:: `Upstream issue #122
         <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
+        Because of this the Reppy parser is deprecated.
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 604b5e314..5c5ac4e41 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,7 +1,9 @@
 import logging
 import sys
 from abc import ABCMeta, abstractmethod
+from warnings import warn
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
@@ -79,6 +81,7 @@ class PythonRobotParser(RobotParser):
 
 class ReppyRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
+        warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
         from reppy.robots import Robots
 
         self.spider = spider
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 276c62a87..fdea46a3d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -235,8 +235,15 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
     if not {"level", "msg", "args"} <= set(logkws):
         warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning)
 
+    if "format" in logkws:
+        warnings.warn(
+            "`format` key in LogFormatter methods has been "
+            "deprecated, use `msg` instead",
+            ScrapyDeprecationWarning,
+        )
+
     level = logkws.get("level", logging.INFO)
-    message = logkws.get("msg")
+    message = logkws.get("format", logkws.get("msg"))
     # NOTE: This also handles 'args' being an empty dict, that case doesn't
     # play well in logger.log calls
     args = logkws if not logkws.get("args") else logkws["args"]
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index be51b0bee..c540d6278 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -55,6 +55,25 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
     return f"{status_int} {to_unicode(message)}"
 
 
+@deprecated
+def response_httprepr(response: Response) -> bytes:
+    """Return raw HTTP representation (as bytes) of the given response. This
+    is provided only for reference, since it's not the exact stream of bytes
+    that was received (that's not exposed by Twisted).
+    """
+    values = [
+        b"HTTP/1.1 ",
+        to_bytes(str(response.status)),
+        b" ",
+        to_bytes(http.RESPONSES.get(response.status, b"")),
+        b"\r\n",
+    ]
+    if response.headers:
+        values.extend([response.headers.to_string(), b"\r\n"])
+    values.extend([b"\r\n", response.body])
+    return b"".join(values)
+
+
 def open_in_browser(
     response: Union[
         "scrapy.http.response.html.HtmlResponse",
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 766b96521..39dfe9ab5 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -6,8 +6,9 @@ from scrapy.downloadermiddlewares.stats import DownloaderStats
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
+from scrapy.utils.response import response_httprepr
 from scrapy.utils.test import get_crawler
-from scrapy.utils.python import to_bytes
+
 
 class MyException(Exception):
     pass
@@ -55,7 +56,7 @@ class TestDownloaderStats(TestCase):
             self.mw.process_response(self.req, test_response, self.spider)
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-                resp_size = to_bytes(test_response)
+                resp_size = len(response_httprepr(test_response))
             self.assertStatsEqual("downloader/response_bytes", resp_size)
 
     def test_process_exception(self):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index d45358d9a..80e15a60f 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -10,6 +10,7 @@ from scrapy.utils.response import (
     get_base_url,
     get_meta_refresh,
     open_in_browser,
+    response_httprepr,
     response_status_message,
 )
 
@@ -19,6 +20,35 @@ __doctests__ = ["scrapy.utils.response"]
 class ResponseUtilsTest(unittest.TestCase):
     dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
 
+    def test_response_httprepr(self):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+
+            r1 = Response("http://www.example.com")
+            self.assertEqual(response_httprepr(r1), b"HTTP/1.1 200 OK\r\n\r\n")
+
+            r1 = Response(
+                "http://www.example.com",
+                status=404,
+                headers={"Content-type": "text/html"},
+                body=b"Some body",
+            )
+            self.assertEqual(
+                response_httprepr(r1),
+                b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body",
+            )
+
+            r1 = Response(
+                "http://www.example.com",
+                status=6666,
+                headers={"Content-type": "text/html"},
+                body=b"Some body",
+            )
+            self.assertEqual(
+                response_httprepr(r1),
+                b"HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body",
+            )
+
     def test_open_in_browser(self):
         url = "http:///www.example.com/some/page.html"
         body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"