Repository: scrapy/scrapy (branch: master)
Commit date: 2023-11-27T19:27:07Z

Workflow file: .github/workflows/checks.yml (Checks)
name: Checks
on: [push, pull_request]

concurrency:
  group: ${{github.workflow}}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  checks:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    strategy:
      fail-fast: false
      matrix:
        include:
        - python-version: "3.12"
          env:
            TOXENV: pylint
        - python-version: 3.8
          env:
            TOXENV: typing
        - python-version: "3.11"  # Keep in sync with .readthedocs.yml
          env:
            TOXENV: docs
        - python-version: "3.12"
          env:
            TOXENV: twinecheck

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Run check
      env: ${{ matrix.env }}
      run: |
        pip install -U tox
        tox

  pre-commit:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    steps:
    - uses: actions/checkout@v4
    - uses: pre-commit/action@v3.0.0

env:
  PIP_INDEX_URL: http://localhost:8629/2023-11-27
  UV_INDEX_URL: http://localhost:8629/2023-11-27


Failed build logs:
step_name: pre-commit/3_Run pre-commitaction@v3.0.0.txt
log: ##[group]Run pre-commit/action@v3.0.0
with:
  extra_args: --all-files
##[endgroup]
##[group]Run python -m pip install pre-commit
[36;1mpython -m pip install pre-commit[0m
shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
##[endgroup]
Defaulting to user installation because normal site-packages is not writeable
Collecting pre-commit
  Downloading pre_commit-3.5.0-py2.py3-none-any.whl (203 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 203.7/203.7 KB 5.6 MB/s eta 0:00:00
Collecting virtualenv>=20.10.0
  Downloading virtualenv-20.24.7-py3-none-any.whl (3.8 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.8/3.8 MB 16.4 MB/s eta 0:00:00
Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from pre-commit) (5.4.1)
Collecting identify>=1.0.0
  Downloading identify-2.5.32-py2.py3-none-any.whl (98 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 98.9/98.9 KB 30.7 MB/s eta 0:00:00
Collecting cfgv>=2.0.0
  Downloading cfgv-3.4.0-py2.py3-none-any.whl (7.2 kB)
Collecting nodeenv>=0.11.1
  Downloading nodeenv-1.8.0-py2.py3-none-any.whl (22 kB)
Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from nodeenv>=0.11.1->pre-commit) (59.6.0)
Collecting platformdirs<5,>=3.9.1
  Downloading platformdirs-4.0.0-py3-none-any.whl (17 kB)
Collecting filelock<4,>=3.12.2
  Downloading filelock-3.13.1-py3-none-any.whl (11 kB)
Collecting distlib<1,>=0.3.7
  Downloading distlib-0.3.7-py2.py3-none-any.whl (468 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 468.9/468.9 KB 28.3 MB/s eta 0:00:00
Installing collected packages: distlib, platformdirs, nodeenv, identify, filelock, cfgv, virtualenv, pre-commit
Successfully installed cfgv-3.4.0 distlib-0.3.7 filelock-3.13.1 identify-2.5.32 nodeenv-1.8.0 platformdirs-4.0.0 pre-commit-3.5.0 virtualenv-20.24.7
##[group]Run python -m pip freeze --local
[36;1mpython -m pip freeze --local[0m
shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
##[endgroup]
argcomplete==3.1.6
cfgv==3.4.0
distlib==0.3.7
filelock==3.13.1
identify==2.5.32
nodeenv==1.8.0
packaging==23.2
pipx==1.2.1
platformdirs==4.0.0
pre-commit==3.5.0
userpath==1.9.1
virtualenv==20.24.7
##[group]Run actions/cache@v3
with:
  path: ~/.cache/pre-commit
  key: pre-commit-3||7a8fe885594aed9a90fd5938b4bb49b65732538a44c08aad3d6ea69d9d0cf64c
  enableCrossOsArchive: false
  fail-on-cache-miss: false
  lookup-only: false
##[endgroup]
Cache Size: ~33 MB (34938818 B)
[command]/usr/bin/tar -xf /home/runner/work/_temp/a467b59a-7374-4fce-b8ee-234d26fbaca7/cache.tzst -P -C /home/runner/work/scrapy/scrapy --use-compress-program unzstd
Received 34938818 of 34938818 (100.0%), 33.3 MBs/sec
Cache restored successfully
Cache restored from key: pre-commit-3||7a8fe885594aed9a90fd5938b4bb49b65732538a44c08aad3d6ea69d9d0cf64c
##[group]Run pre-commit run --show-diff-on-failure --color=always --all-files
[36;1mpre-commit run --show-diff-on-failure --color=always --all-files[0m
shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
##[endgroup]
bandit...................................................................[42mPassed[m
flake8...................................................................[42mPassed[m
black....................................................................[41mFailed[m
[2m- hook id: black[m
[2m- files were modified by this hook[m

[1mreformatted scrapy/cmdline.py[0m

[1mAll done! ✨ 🍰 ✨[0m
[34m[1m1 file [0m[1mreformatted[0m, [34m340 files [0mleft unchanged.

isort....................................................................[42mPassed[m
blacken-docs.............................................................[42mPassed[m
pre-commit hook(s) made changes.
If you are seeing this message in CI, reproduce locally with: `pre-commit run --all-files`.
To run `pre-commit` as part of git workflow, use `pre-commit install`.
All changes made by hooks:
[1mdiff --git a/scrapy/cmdline.py b/scrapy/cmdline.py[m
[1mindex 761f9df..69e4f79 100644[m
[1m--- a/scrapy/cmdline.py[m
[1m+++ b/scrapy/cmdline.py[m
[36m@@ -72,7 +72,7 @@[m [mdef _get_commands_dict(settings, inproject):[m
 [m
 def _pop_command_name(argv):[m
     for i, v in enumerate(argv):[m
[31m-        if v.startswith('-'):[m
[32m+[m[32m        if v.startswith("-"):[m
             continue[m
         return argv.pop(i)[m
 [m
##[error]Process completed with exit code 1.


Changed files:
docs/topics/request-response.rst
scrapy/cmdline.py

Fix diff:
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index adf3d0f4a..8edf710bc 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -193,18 +193,47 @@ Request objects
         :meth:`replace`.
 
     .. attribute:: Request.meta
-
-        A dict that contains arbitrary metadata for this request. This dict is
-        empty for new Requests, and is usually  populated by different Scrapy
-        components (extensions, middlewares, etc). So the data contained in this
-        dict depends on the extensions you have enabled.
-
-        See :ref:`topics-request-meta` for a list of special meta keys
-        recognized by Scrapy.
-
-        This dict is :doc:`shallow copied <library/copy>` when the request is
-        cloned using the ``copy()`` or ``replace()`` methods, and can also be
-        accessed, in your spider, from the ``response.meta`` attribute.
+       :value: {}
+
+        A dictionary of arbitrary metadata for the request.
+
+        You may extend request metadata as you see fit.
+
+        Request metadata can also be accessed through the
+        :attr:`~scrapy.http.Response.meta` attribute of a response.
+
+        To pass data from one spider callback to another, consider using
+        :attr:`cb_kwargs` instead. However, request metadata may be the right
+        choice in certain scenarios, such as to maintain some debugging data
+        across all follow-up requests (e.g. the source URL).
+
+        A common use of request metadata is to define request-specific
+        parameters for Scrapy components (extensions, middlewares, etc.). For
+        example, if you set ``dont_retry`` to ``True``,
+        :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never
+        retry that request, even if it fails. See :ref:`topics-request-meta`.
+
+        You may also use request metadata in your custom Scrapy components, for
+        example, to keep request state information relevant to your component.
+        For example,
+        :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the
+        ``retry_times`` metadata key to keep track of how many times a request
+        has been retried so far.
+
+        Copying all the metadata of a previous request into a new, follow-up
+        request in a spider callback is a bad practice, because request
+        metadata may include metadata set by Scrapy components that is not
+        meant to be copied into other requests. For example, copying the
+        ``retry_times`` metadata key into follow-up requests can lower the
+        amount of retries allowed for those follow-up requests.
+
+        You should only copy all request metadata from one request to another
+        if the new request is meant to replace the old request, as is often the
+        case when returning a request from a :ref:`downloader middleware
+        <topics-downloader-middleware>` method.
+
+        Also mind that the :meth:`copy` and :meth:`replace` request methods
+        :doc:`shallow-copy <library/copy>` request metadata.
 
     .. attribute:: Request.cb_kwargs
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 761f9df83..6580ba9ce 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -71,10 +71,12 @@ def _get_commands_dict(settings, inproject):
 
 
 def _pop_command_name(argv):
-    for i, v in enumerate(argv):
-        if v.startswith('-'):
-            continue
-        return argv.pop(i)
+    i = 0
+    for arg in argv[1:]:
+        if not arg.startswith("-"):
+            del argv[i]
+            return arg
+        i += 1
 
 
 def _print_header(settings, inproject):
@@ -86,11 +88,12 @@ def _print_header(settings, inproject):
         print(f"Scrapy {version} - no active project\n")
 
 
-def _print_commands(settings, inproject, cmds):
+def _print_commands(settings, inproject):
     _print_header(settings, inproject)
     print("Usage:")
     print("  scrapy <command> [options] [args]\n")
     print("Available commands:")
+    cmds = _get_commands_dict(settings, inproject)
     for cmdname, cmdclass in sorted(cmds.items()):
         print(f"  {cmdname:<13} {cmdclass.short_desc()}")
     if not inproject:
@@ -106,7 +109,6 @@ def _print_unknown_command(settings, cmdname, inproject):
     print('Use "scrapy" to see available commands')
 
 
-# TODO: Confusion, can be improved.
 def _run_print_help(parser, func, *a, **kw):
     try:
         func(*a, **kw)
@@ -134,9 +136,9 @@ def execute(argv=None, settings=None):
 
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
-    cmdname = _pop_command_name(argv[1:])
+    cmdname = _pop_command_name(argv)
     if not cmdname:
-        _print_commands(settings, inproject, cmds)
+        _print_commands(settings, inproject)
         sys.exit(0)
     elif cmdname not in cmds:
         _print_unknown_command(settings, cmdname, inproject)
@@ -152,7 +154,7 @@ def execute(argv=None, settings=None):
     settings.setdict(cmd.default_settings, priority="command")
     cmd.settings = settings
     cmd.add_options(parser)
-    opts, args = parser.parse_known_args(argv[1:])
+    opts, args = parser.parse_known_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
     cmd.crawler_process = CrawlerProcess(settings)