Repository: tox-dev/tox

Issue:
Can't append deps to testenv via command line
## Issue

The manual indicates that I should be able to add a dependency via something like:
```
tox -e test --x testenv.deps+=debugpy
```

which would append debugpy to the dependencies specified in my tox.ini testenv section.

However, this results in an error, as below

## Environment

Provide at least:

- OS: Mac OS

<details open>
<summary>Output of <code>pip list</code> of the host Python, where <code>tox</code> is installed</summary>

```console
Package                   Version
------------------------- ------------
aiohttp                   3.9.1
aiohttp-cors              0.7.0
aiosignal                 1.3.1
annotated-types           0.6.0
anyio                     4.2.0
archspec                  0.2.1
astroid                   3.0.3
asttokens                 2.4.1
asv                       0.6.3
asv_runner                0.2.1
attrs                     23.1.0
autopep8                  2.0.4
awscli                    1.32.6
bcrypt                    4.1.2
black                     23.12.0
blessed                   1.20.0
boltons                   23.0.0
boto3                     1.34.6
botocore                  1.34.6
Brotli                    1.0.9
build                     1.0.3
cachetools                5.3.2
cattrs                    23.2.3
certifi                   2023.11.17
cffi                      1.15.1
chardet                   5.2.0
charset-normalizer        2.0.4
click                     8.1.7
cloudpickle               3.0.0
cmake-language-server     0.1.9
colorama                  0.4.6
colorful                  0.5.6
conda                     23.11.0
conda-content-trust       0.2.0
conda-libmamba-solver     23.11.1
conda-package-handling    2.2.0
conda_package_streaming   0.9.0
contourpy                 1.2.0
cryptography              41.0.3
cycler                    0.12.1
Cython                    3.0.8
daft                      0.1.2
decorator                 5.1.1
dill                      0.3.8
distlib                   0.3.8
distro                    1.8.0
docstring-to-markdown     0.15
docutils                  0.16
dplutils                  0.3.0
duckdb                    0.9.2
econml                    0.15.0
executing                 2.0.1
fastapi                   0.108.0
filelock                  3.13.1
flake8                    7.0.0
fonttools                 4.49.0
frozenlist                1.4.0
fsspec                    2024.2.0
getdaft                   0.2.15
globus-sdk                3.37.0
google-api-core           2.15.0
google-auth               2.27.0
googleapis-common-protos  1.62.0
gpustat                   1.1.1
grpcio                    1.60.0
h11                       0.14.0
httpcore                  1.0.4
httpx                     0.27.0
idna                      3.4
importlib-metadata        7.0.0
iniconfig                 1.1.1
ipython                   8.18.1
isort                     5.13.2
jax                       0.4.24
jaxlib                    0.4.24
jedi                      0.19.1
jmespath                  1.0.1
joblib                    1.3.2
json5                     0.9.24
jsonpatch                 1.32
jsonpointer               2.1
jsonschema                4.21.1
jsonschema-specifications 2023.12.1
kiwisolver                1.4.5
libmambapy                1.5.3
lightgbm                  4.3.0
llvmlite                  0.42.0
lsprotocol                2023.0.1
matplotlib                3.8.3
matplotlib-inline         0.1.6
mccabe                    0.7.0
menuinst                  2.0.1
ml-dtypes                 0.3.2
msgpack                   1.0.7
multidict                 6.0.4
mypy-extensions           1.0.0
netifaces                 0.11.0
networkx                  3.2.1
numba                     0.59.0
numpy                     1.26.2
nvidia-ml-py              12.535.133
opencensus                0.11.4
opencensus-context        0.1.3
opt-einsum                3.3.0
packaging                 24.0
pandas                    2.1.4
paramiko                  3.4.0
parsl                     2024.2.12
parso                     0.8.3
pathspec                  0.12.1
patsy                     0.5.6
pexpect                   4.9.0
pillow                    10.2.0
pip                       23.3
platformdirs              4.1.0
pluggy                    1.3.0
prometheus-client         0.19.0
prompt-toolkit            3.0.43
protobuf                  3.20.3
psutil                    5.9.8
ptvsd                     4.3.2
ptyprocess                0.7.0
pure-eval                 0.2.2
py-spy                    0.3.14
py4j                      0.10.9.5
pyarrow                   14.0.2
pyasn1                    0.5.1
pyasn1-modules            0.3.0
pycodestyle               2.11.1
pycosat                   0.6.6
pycparser                 2.21
pydantic_core             2.14.6
pydocstyle                6.3.0
pyflakes                  3.2.0
pygls                     1.3.0
Pygments                  2.17.2
PyJWT                     2.8.0
pylint                    3.0.4
Pympler                   1.0.1
PyNaCl                    1.5.0
pyOpenSSL                 23.2.0
pyparsing                 3.1.1
pyproject-api             1.6.1
pyproject_hooks           1.0.0
PySocks                   1.7.1
pyspark                   3.3.2
pytest                    7.4.0
python-dateutil           2.8.2
python-lsp-jsonrpc        1.1.2
python-lsp-server         1.10.0
pytoolconfig              1.2.6
pytz                      2023.3.post1
PyYAML                    6.0.1
pyzmq                     25.1.2
ray                       2.9.1
raydp                     1.6.0
referencing               0.32.1
requests                  2.31.0
rope                      1.11.0
rpds-py                   0.17.1
rsa                       4.7.2
ruamel.yaml               0.17.21
ruff                      0.1.8
s3transfer                0.10.0
scikit-learn              1.4.1.post1
scipy                     1.12.0
setproctitle              1.3.3
setuptools                68.0.0
shap                      0.43.0
six                       1.16.0
slicer                    0.0.7
smart-open                6.4.0
snakeviz                  2.2.0
sniffio                   1.3.0
snowballstemmer           2.2.0
sparse                    0.15.1
stack-data                0.6.3
starlette                 0.32.0.post1
statsmodels               0.14.1
tabulate                  0.9.0
tblib                     3.0.0
threadpoolctl             3.3.0
tomli                     2.0.1
tomlkit                   0.12.4
tornado                   6.4
tox                       4.14.2
tqdm                      4.65.0
traitlets                 5.14.0
truststore                0.8.0
typeguard                 4.1.5
typing_extensions         4.9.0
tzdata                    2023.3
ujson                     5.9.0
urllib3                   1.26.18
uvicorn                   0.25.0
virtualenv                20.25.0
wcwidth                   0.2.12
websockets                12.0
whatthepatch              1.0.5
wheel                     0.41.2
yapf                      0.40.2
yarl                      1.9.4
zipp                      3.17.0
zstandard                 0.19.0

```

</details>

## Output of running tox

<details open>
<summary>Output of <code>tox -rvv</code></summary>

```console

```

</details>

## Minimal example

<!--  If possible, provide a minimal reproducer for the issue. -->

```console
-->$ tox -e test -r --override testenv.deps+=debugpy

test: remove tox env folder /Users/arik/ws/ssec/dplutils/.tox/test
.pkg: remove tox env folder /Users/arik/ws/ssec/dplutils/.tox/.pkg
test: internal error
Traceback (most recent call last):
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/session/cmd/run/single.py", line 47, in _evaluate
    tox_env.setup()
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/tox_env/api.py", line 248, in setup
    self._setup_env()
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/tox_env/python/runner.py", line 109, in _setup_env
    self._install_deps()
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/tox_env/python/runner.py", line 112, in _install_deps
    requirements_file: PythonDeps = self.conf["deps"]
                                    ~~~~~~~~~^^^^^^^^
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/config/sets.py", line 116, in __getitem__
    return self.load(item)
           ^^^^^^^^^^^^^^^
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/config/sets.py", line 127, in load
    return config_definition.__call__(self._conf, self.loaders, ConfigLoadArgs(chain, self.name, self.env_name))  # noqa: PLC2801
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/config/of_type.py", line 103, in __call__
    value = loader.load(key, self.of_type, self.factory, conf, args)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/arik/.miniconda3/lib/python3.11/site-packages/tox/config/loader/api.py", line 153, in load
    raise ValueError(msg)
ValueError: Only able to append to lists and dicts
  test: FAIL code 2 (0.14 seconds)
  evaluation failed :( (0.17 seconds)

```



Discussion:
It is one dash for the short form of override, not two, but that does not seem the problem...

```
tox -e test --x testenv.deps+=debugpy
```
...as you use the long form in your reproducer.

```
--override
```

I wonder whether the `+=` of 

```
--override testenv.deps+=debugpy
```

is the issue.

I am currently not near a computer and cannot test this.

I just found this PR by @stefanor which added support for the += syntax
https://github.com/tox-dev/tox/pull/3088/files

The issue should be somewhere along those changed files.
@jugmac00, thanks, yeah not sure how the double-dash typo got in, I did use either of `-x` or `--override`. I think I know where the issue is and a way to resolve it (issue being that deps is `PythonDeps` type and doesn't have a specific handler at https://github.com/tox-dev/tox/blob/main/src/tox/config/loader/api.py#L149 like SetEnv), I can work on a PR if that is OK.

At the same time I was hoping to expand the scope a slight bit, since it could probably be handled in a more general way by first appending to the raw value and then processing (e.g. `-x` would modify the config, as opposed to values) - also I notice that it looks like multiple overrides _should_ be possible, but only the last one is taken, and the handling of that would touch code in the same region and affect the design slightly.
In tox, `tox -e py311 -x testenv.deps+=debugpy` works for me. Maybe the issue is that your deps aren't in the format usually expected? What does your `tox.ini` look like?
@stefanor, could you confirm that this also works for you when you have the setting set in `tox.ini`? Or is this equivalent to `testenv.deps=debugpy`? 
Aha, when it's set, it's a `tox.tox_env.python.pip.req_file.PythonDeps`



Patch:
diff --git a/docs/changelog/3256.bugfix.rst b/docs/changelog/3256.bugfix.rst
new file mode 100644
index 000000000..819082c70
--- /dev/null
+++ b/docs/changelog/3256.bugfix.rst
@@ -0,0 +1,1 @@
+Allow appending to ``deps`` with ``--override testenv.deps+=foo`` - by :user:`stefanor`.
diff --git a/src/tox/config/loader/api.py b/src/tox/config/loader/api.py
index f02361dea..98a235ed6 100644
--- a/src/tox/config/loader/api.py
+++ b/src/tox/config/loader/api.py
@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING, Any, List, Mapping, TypeVar
 
 from tox.plugin import impl
+from tox.tox_env.python.pip.req_file import PythonDeps
 
 from .convert import Convert, Factory
 from .str_convert import StrConvert
@@ -148,6 +149,8 @@ def load(  # noqa: PLR0913
                 converted.update(converted_override)
             elif isinstance(converted, SetEnv) and isinstance(converted_override, SetEnv):
                 converted.update(converted_override, override=True)
+            elif isinstance(converted, PythonDeps) and isinstance(converted_override, PythonDeps):
+                converted += converted_override
             else:
                 msg = "Only able to append to lists and dicts"
                 raise ValueError(msg)
diff --git a/src/tox/tox_env/python/pip/req_file.py b/src/tox/tox_env/python/pip/req_file.py
index 24e3fac2d..c88e5cd17 100644
--- a/src/tox/tox_env/python/pip/req_file.py
+++ b/src/tox/tox_env/python/pip/req_file.py
@@ -118,6 +118,10 @@ def unroll(self) -> tuple[list[str], list[str]]:
             self._unroll = result_opts, result_req
         return self._unroll
 
+    def __iadd__(self, other: PythonDeps) -> PythonDeps:  # noqa: PYI034
+        self._raw += "\n" + other._raw
+        return self
+
     @classmethod
     def factory(cls, root: Path, raw: object) -> PythonDeps:
         if not isinstance(raw, str):


Test patch:
diff --git a/tests/config/test_main.py b/tests/config/test_main.py
index 0e8728779..264e2e1df 100644
--- a/tests/config/test_main.py
+++ b/tests/config/test_main.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from functools import partial
 from pathlib import Path
 from typing import TYPE_CHECKING, List
 
@@ -9,6 +10,7 @@
 from tox.config.loader.api import Override
 from tox.config.loader.memory import MemoryLoader
 from tox.config.sets import ConfigSet
+from tox.tox_env.python.pip.req_file import PythonDeps
 
 if TYPE_CHECKING:
     from tests.conftest import ToxIniCreator
@@ -98,6 +100,37 @@ def test_config_override_appends_to_empty_setenv(tox_ini_conf: ToxIniCreator) ->
     assert conf["setenv"].load("foo") == "bar"
 
 
+def test_config_override_appends_to_pythondeps(tox_ini_conf: ToxIniCreator, tmp_path: Path) -> None:
+    example = """
+    [testenv]
+    deps = foo
+    """
+    conf = tox_ini_conf(example, override=[Override("testenv.deps+=bar")]).get_env("testenv")
+    conf.add_config(
+        "deps",
+        of_type=PythonDeps,
+        factory=partial(PythonDeps.factory, tmp_path),
+        default=PythonDeps("", root=tmp_path),
+        desc="desc",
+    )
+    assert conf["deps"].lines() == ["foo", "bar"]
+
+
+def test_config_override_appends_to_empty_pythondeps(tox_ini_conf: ToxIniCreator, tmp_path: Path) -> None:
+    example = """
+    [testenv]
+    """
+    conf = tox_ini_conf(example, override=[Override("testenv.deps+=bar")]).get_env("testenv")
+    conf.add_config(
+        "deps",
+        of_type=PythonDeps,
+        factory=partial(PythonDeps.factory, tmp_path),
+        default=PythonDeps("", root=tmp_path),
+        desc="desc",
+    )
+    assert conf["deps"].lines() == ["bar"]
+
+
 def test_config_override_cannot_append(tox_ini_conf: ToxIniCreator) -> None:
     example = """
     [testenv]
diff --git a/tests/tox_env/python/pip/test_req_file.py b/tests/tox_env/python/pip/test_req_file.py
index 39a640e4b..50a638b3a 100644
--- a/tests/tox_env/python/pip/test_req_file.py
+++ b/tests/tox_env/python/pip/test_req_file.py
@@ -67,3 +67,10 @@ def test_opt_only_req_file(tmp_path: Path) -> None:
     python_deps = PythonDeps(raw="-rr.txt", root=tmp_path)
     assert not python_deps.requirements
     assert python_deps.options == Namespace(features_enabled=["fast-deps"])
+
+
+def test_req_iadd(tmp_path: Path) -> None:
+    a = PythonDeps(raw="foo", root=tmp_path)
+    b = PythonDeps(raw="bar", root=tmp_path)
+    a += b
+    assert a.lines() == ["foo", "bar"]