Repository: tox-dev/tox

Issue:
Multiple env subsitutions broken in tox 4.3
## Issue

Having multiple `{env}` substitutions, as in
```
commands =
    echo "{env:USER}" "{env:HOME}" "{env:USER}"
```
fails with `circular chain between set env USER, HOME` in tox version 4.3.0 and 4.3.1. This worked fine in tox version 4.2.8 and earlier. The issue also appears if the multiple `{env}` substitutions are spread accross multiple commands, as in
```
commands =
    echo "{env:USER}" "{env:HOME}"
    echo "{env:USER}"
```

Expected behavior is that this executes without complaint, as in tox version 4.2.8.

## Environment

Provide at least:

- OS: Ubuntu 20.04.5
- `pip list` of the host Python where `tox` is installed:

```console
Package       Version
------------- -------
cachetools    5.2.1
chardet       5.1.0
colorama      0.4.6
distlib       0.3.6
filelock      3.9.0
packaging     23.0
pip           22.3.1
pkg_resources 0.0.0
platformdirs  2.6.2
pluggy        1.0.0
pyproject_api 1.4.0
setuptools    44.0.0
tomli         2.0.1
tox           4.3.1
virtualenv    20.17.1
wheel         0.38.4
```

## Output of running tox

Provide the output of `tox -rvv`:

```console
py: 100 I find interpreter for spec PythonSpec(path=/tmp/test/venv/bin/python3) [virtualenv/discovery/builtin.py:56]
py: 100 D discover exe for PythonInfo(spec=CPython3.8.10.final.0-64, exe=/tmp/test/venv/bin/python3, platform=linux, version='3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]', encoding_fs_io=utf-8-utf-8) in /usr [virtualenv/discovery/py_info.py:437]
py: 101 D filesystem is case-sensitive [virtualenv/info.py:24]
py: 101 D got python info of /usr/bin/python3.8 from /home/von/.local/share/virtualenv/py_info/1/df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6.json [virtualenv/app_data/via_disk_folder.py:129]
py: 102 I proposed PythonInfo(spec=CPython3.8.10.final.0-64, system=/usr/bin/python3.8, exe=/tmp/test/venv/bin/python3, platform=linux, version='3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]', encoding_fs_io=utf-8-utf-8) [virtualenv/discovery/builtin.py:63]
py: 102 D accepted PythonInfo(spec=CPython3.8.10.final.0-64, system=/usr/bin/python3.8, exe=/tmp/test/venv/bin/python3, platform=linux, version='3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]', encoding_fs_io=utf-8-utf-8) [virtualenv/discovery/builtin.py:65]
py: 117 I create virtual environment via CPython3Posix(dest=/tmp/test/.tox/py, clear=False, no_vcs_ignore=False, global=False) [virtualenv/run/session.py:48]
py: 117 D create folder /tmp/test/.tox/py/bin [virtualenv/util/path/_sync.py:9]
py: 118 D create folder /tmp/test/.tox/py/lib/python3.8/site-packages [virtualenv/util/path/_sync.py:9]
py: 118 D write /tmp/test/.tox/py/pyvenv.cfg [virtualenv/create/pyenv_cfg.py:30]
py: 118 D 	home = /usr/bin [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	implementation = CPython [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	version_info = 3.8.10.final.0 [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	virtualenv = 20.17.1 [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	include-system-site-packages = false [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	base-prefix = /usr [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	base-exec-prefix = /usr [virtualenv/create/pyenv_cfg.py:34]
py: 118 D 	base-executable = /usr/bin/python3.8 [virtualenv/create/pyenv_cfg.py:34]
py: 118 D symlink /usr/bin/python3.8 to /tmp/test/.tox/py/bin/python [virtualenv/util/path/_sync.py:28]
py: 118 D create virtualenv import hook file /tmp/test/.tox/py/lib/python3.8/site-packages/_virtualenv.pth [virtualenv/create/via_global_ref/api.py:89]
py: 118 D create /tmp/test/.tox/py/lib/python3.8/site-packages/_virtualenv.py [virtualenv/create/via_global_ref/api.py:92]
py: 118 D ============================== target debug ============================== [virtualenv/run/session.py:50]
py: 119 D debug via /tmp/test/.tox/py/bin/python /tmp/test/venv/lib/python3.8/site-packages/virtualenv/create/debug.py [virtualenv/create/creator.py:197]
py: 119 D {
  "sys": {
    "executable": "/tmp/test/.tox/py/bin/python",
    "_base_executable": "/tmp/test/.tox/py/bin/python",
    "prefix": "/tmp/test/.tox/py",
    "base_prefix": "/usr",
    "real_prefix": null,
    "exec_prefix": "/tmp/test/.tox/py",
    "base_exec_prefix": "/usr",
    "path": [
      "/usr/lib/python38.zip",
      "/usr/lib/python3.8",
      "/usr/lib/python3.8/lib-dynload",
      "/tmp/test/.tox/py/lib/python3.8/site-packages"
    ],
    "meta_path": [
      "<class '_virtualenv._Finder'>",
      "<class '_frozen_importlib.BuiltinImporter'>",
      "<class '_frozen_importlib.FrozenImporter'>",
      "<class '_frozen_importlib_external.PathFinder'>"
    ],
    "fs_encoding": "utf-8",
    "io_encoding": "utf-8"
  },
  "version": "3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]",
  "makefile_filename": "/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/Makefile",
  "os": "<module 'os' from '/usr/lib/python3.8/os.py'>",
  "site": "<module 'site' from '/usr/lib/python3.8/site.py'>",
  "datetime": "<module 'datetime' from '/usr/lib/python3.8/datetime.py'>",
  "math": "<module 'math' (built-in)>",
  "json": "<module 'json' from '/usr/lib/python3.8/json/__init__.py'>"
} [virtualenv/run/session.py:51]
py: 135 I add seed packages via FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/von/.local/share/virtualenv) [virtualenv/run/session.py:55]
py: 137 D got embed update of distribution setuptools from /home/von/.local/share/virtualenv/wheel/3.8/embed/3/setuptools.json [virtualenv/app_data/via_disk_folder.py:129]
py: 137 D got embed update of distribution wheel from /home/von/.local/share/virtualenv/wheel/3.8/embed/3/wheel.json [virtualenv/app_data/via_disk_folder.py:129]
py: 137 D got embed update of distribution pip from /home/von/.local/share/virtualenv/wheel/3.8/embed/3/pip.json [virtualenv/app_data/via_disk_folder.py:129]
py: 139 D install setuptools from wheel /tmp/test/venv/lib/python3.8/site-packages/virtualenv/seed/wheels/embed/setuptools-65.6.3-py3-none-any.whl via CopyPipInstall [virtualenv/seed/embed/via_app_data/via_app_data.py:47]
py: 139 D install wheel from wheel /tmp/test/venv/lib/python3.8/site-packages/virtualenv/seed/wheels/embed/wheel-0.38.4-py3-none-any.whl via CopyPipInstall [virtualenv/seed/embed/via_app_data/via_app_data.py:47]
py: 139 D install pip from wheel /tmp/test/venv/lib/python3.8/site-packages/virtualenv/seed/wheels/embed/pip-22.3.1-py3-none-any.whl via CopyPipInstall [virtualenv/seed/embed/via_app_data/via_app_data.py:47]
py: 140 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/pip-22.3.1-py3-none-any/pip-22.3.1.dist-info to /tmp/test/.tox/py/lib/python3.8/site-packages/pip-22.3.1.dist-info [virtualenv/util/path/_sync.py:36]
py: 140 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/setuptools-65.6.3-py3-none-any/_distutils_hack to /tmp/test/.tox/py/lib/python3.8/site-packages/_distutils_hack [virtualenv/util/path/_sync.py:36]
py: 140 D copy /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/wheel-0.38.4-py3-none-any/wheel-0.38.4.virtualenv to /tmp/test/.tox/py/lib/python3.8/site-packages/wheel-0.38.4.virtualenv [virtualenv/util/path/_sync.py:36]
py: 141 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/wheel-0.38.4-py3-none-any/wheel-0.38.4.dist-info to /tmp/test/.tox/py/lib/python3.8/site-packages/wheel-0.38.4.dist-info [virtualenv/util/path/_sync.py:36]
py: 141 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/setuptools-65.6.3-py3-none-any/pkg_resources to /tmp/test/.tox/py/lib/python3.8/site-packages/pkg_resources [virtualenv/util/path/_sync.py:36]
py: 142 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/pip-22.3.1-py3-none-any/pip to /tmp/test/.tox/py/lib/python3.8/site-packages/pip [virtualenv/util/path/_sync.py:36]
py: 142 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/wheel-0.38.4-py3-none-any/wheel to /tmp/test/.tox/py/lib/python3.8/site-packages/wheel [virtualenv/util/path/_sync.py:36]
py: 147 D generated console scripts wheel3 wheel3.8 wheel wheel-3.8 [virtualenv/seed/embed/via_app_data/pip_install/base.py:41]
py: 150 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/setuptools-65.6.3-py3-none-any/setuptools to /tmp/test/.tox/py/lib/python3.8/site-packages/setuptools [virtualenv/util/path/_sync.py:36]
py: 171 D copy /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/setuptools-65.6.3-py3-none-any/setuptools-65.6.3.virtualenv to /tmp/test/.tox/py/lib/python3.8/site-packages/setuptools-65.6.3.virtualenv [virtualenv/util/path/_sync.py:36]
py: 172 D copy /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/setuptools-65.6.3-py3-none-any/distutils-precedence.pth to /tmp/test/.tox/py/lib/python3.8/site-packages/distutils-precedence.pth [virtualenv/util/path/_sync.py:36]
py: 172 D copy directory /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/setuptools-65.6.3-py3-none-any/setuptools-65.6.3.dist-info to /tmp/test/.tox/py/lib/python3.8/site-packages/setuptools-65.6.3.dist-info [virtualenv/util/path/_sync.py:36]
py: 173 D generated console scripts  [virtualenv/seed/embed/via_app_data/pip_install/base.py:41]
py: 188 D copy /home/von/.local/share/virtualenv/wheel/3.8/image/1/CopyPipInstall/pip-22.3.1-py3-none-any/pip-22.3.1.virtualenv to /tmp/test/.tox/py/lib/python3.8/site-packages/pip-22.3.1.virtualenv [virtualenv/util/path/_sync.py:36]
py: 188 D generated console scripts pip3.8 pip-3.8 pip pip3 [virtualenv/seed/embed/via_app_data/pip_install/base.py:41]
py: 188 I add activators for Bash, CShell, Fish, Nushell, PowerShell, Python [virtualenv/run/session.py:61]
py: 189 D write /tmp/test/.tox/py/pyvenv.cfg [virtualenv/create/pyenv_cfg.py:30]
py: 189 D 	home = /usr/bin [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	implementation = CPython [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	version_info = 3.8.10.final.0 [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	virtualenv = 20.17.1 [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	include-system-site-packages = false [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	base-prefix = /usr [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	base-exec-prefix = /usr [virtualenv/create/pyenv_cfg.py:34]
py: 189 D 	base-executable = /usr/bin/python3.8 [virtualenv/create/pyenv_cfg.py:34]
py: 190 E internal error [tox/session/cmd/run/single.py:58]
Traceback (most recent call last):
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/__init__.py", line 77, in replacer
    replaced = replace(conf, self, raw_, args_)  # do replacements
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/replace.py", line 57, in replace
    return Replacer(conf, loader, conf_args=args, depth=depth).join(find_replace_expr(value))
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/replace.py", line 178, in join
    return "".join(self(value))
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/replace.py", line 175, in __call__
    return [self._replace_match(me) if isinstance(me, MatchExpression) else str(me) for me in value]
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/replace.py", line 175, in <listcomp>
    return [self._replace_match(me) if isinstance(me, MatchExpression) else str(me) for me in value]
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/replace.py", line 187, in _replace_match
    replace_value = replace_env(self.conf, args, self.conf_args)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/replace.py", line 325, in replace_env
    raise MatchRecursionError(f"circular chain between set env {circular}")
tox.config.loader.ini.replace.MatchRecursionError: circular chain between set env USER, HOME

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 46, in _evaluate
    code, outcomes = run_commands(tox_env, no_test)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 83, in run_commands
    status_main = run_command_set(tox_env, "commands", chdir, ignore_errors, outcomes)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 96, in run_command_set
    command_set: list[Command] = tox_env.conf[key]
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/sets.py", line 114, in __getitem__
    return self.load(item)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/sets.py", line 125, in load
    return config_definition.__call__(self._conf, self.loaders, ConfigLoadArgs(chain, self.name, self.env_name))
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/of_type.py", line 102, in __call__
    value = loader.load(key, self.of_type, self.factory, conf, args)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/api.py", line 126, in load
    with self.build(future, key, of_type, conf, raw, args) as prepared:
  File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__
    return next(self.gen)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/__init__.py", line 87, in build
    raw = replacer(raw, args)
  File "/tmp/test/venv/lib/python3.8/site-packages/tox/config/loader/ini/__init__.py", line 83, in replacer
    raise HandledError(msg) from exception
tox.report.HandledError: replace failed in py.commands with MatchRecursionError('circular chain between set env USER, HOME')
  py: FAIL code 2 (0.10 seconds)
  evaluation failed :( (0.13 seconds)

```

## Minimal example

If possible, provide a minimal reproducer for the issue:

```console
[tox]
requires =
    tox == 4.3.1

[testenv]
passenv =
    USER
    HOME
allowlist_externals =
    echo
commands =
    echo "{env:USER}" "{env:HOME}" "{env:USER}"
```


Discussion:

Patch:
diff --git a/docs/changelog/2869.bugfix.rst b/docs/changelog/2869.bugfix.rst
new file mode 100644
index 00000000..78588787
--- /dev/null
+++ b/docs/changelog/2869.bugfix.rst
@@ -0,0 +1,5 @@
+Fix regression introduced in 4.3.0 which occured when a substitution expression
+for an environment variable that had previously been substituted appears in the
+ini file after a substitution expression for a different environment variable.
+This situation erroneously resulted in an exception about "circular chain
+between set" of those variables - by :user:`masenf`.
diff --git a/src/tox/config/loader/ini/replace.py b/src/tox/config/loader/ini/replace.py
index d856277c..40730ca5 100644
--- a/src/tox/config/loader/ini/replace.py
+++ b/src/tox/config/loader/ini/replace.py
@@ -178,29 +178,32 @@ class Replacer:
         return "".join(self(value))
 
     def _replace_match(self, value: MatchExpression) -> str:
+        # use a copy of conf_args so any changes from this replacement do NOT
+        # affect adjacent substitutions (#2869)
+        conf_args = self.conf_args.copy()
         of_type, *args = flattened_args = [self.join(arg) for arg in value.expr]
         if of_type == "/":
             replace_value: str | None = os.sep
         elif of_type == "" and args == [""]:
             replace_value = os.pathsep
         elif of_type == "env":
-            replace_value = replace_env(self.conf, args, self.conf_args)
+            replace_value = replace_env(self.conf, args, conf_args)
         elif of_type == "tty":
             replace_value = replace_tty(args)
         elif of_type == "posargs":
-            replace_value = replace_pos_args(self.conf, args, self.conf_args)
+            replace_value = replace_pos_args(self.conf, args, conf_args)
         else:
             replace_value = replace_reference(
                 self.conf,
                 self.loader,
                 ARG_DELIMITER.join(flattened_args),
-                self.conf_args,
+                conf_args,
             )
         if replace_value is not None:
             needs_expansion = any(isinstance(m, MatchExpression) for m in find_replace_expr(replace_value))
             if needs_expansion:
                 try:
-                    return replace(self.conf, self.loader, replace_value, self.conf_args, self.depth + 1)
+                    return replace(self.conf, self.loader, replace_value, conf_args, self.depth + 1)
                 except MatchRecursionError as err:
                     LOGGER.warning(str(err))
                     return replace_value


Test patch:
diff --git a/tests/config/loader/ini/replace/test_replace_env_var.py b/tests/config/loader/ini/replace/test_replace_env_var.py
index d2e9fa58..54438b0d 100644
--- a/tests/config/loader/ini/replace/test_replace_env_var.py
+++ b/tests/config/loader/ini/replace/test_replace_env_var.py
@@ -6,7 +6,7 @@ from typing import Generator
 import pytest
 
 from tests.config.loader.ini.replace.conftest import ReplaceOne
-from tox.pytest import MonkeyPatch
+from tox.pytest import LogCaptureFixture, MonkeyPatch
 
 
 def test_replace_env_set(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
@@ -75,6 +75,20 @@ def test_replace_env_missing_default_from_env(replace_one: ReplaceOne, monkeypat
     assert result == "yes"
 
 
+def test_replace_env_var_multiple(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Multiple env substitutions on a single line."""
+    monkeypatch.setenv("MAGIC", "MAGIC")
+    monkeypatch.setenv("TRAGIC", "TRAGIC")
+    result = replace_one("{env:MAGIC} {env:TRAGIC} {env:MAGIC}")
+    assert result == "MAGIC TRAGIC MAGIC"
+
+
+def test_replace_env_var_multiple_default(replace_one: ReplaceOne) -> None:
+    """Multiple env substitutions on a single line with default values."""
+    result = replace_one("{env:MAGIC:foo} {env:TRAGIC:bar} {env:MAGIC:baz}")
+    assert result == "foo bar baz"
+
+
 def test_replace_env_var_circular(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
     """Replacement values will not infinitely loop"""
     monkeypatch.setenv("MAGIC", "{env:MAGIC}")
@@ -97,12 +111,34 @@ def reset_env_var_after_delay(monkeypatch: MonkeyPatch) -> Generator[threading.T
 
 
 @pytest.mark.usefixtures("reset_env_var_after_delay")
-def test_replace_env_var_circular_flip_flop(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+def test_replace_env_var_circular_flip_flop(
+    replace_one: ReplaceOne,
+    monkeypatch: MonkeyPatch,
+    caplog: LogCaptureFixture,
+) -> None:
     """Replacement values will not infinitely loop back and forth"""
     monkeypatch.setenv("TRAGIC", "{env:MAGIC}")
     monkeypatch.setenv("MAGIC", "{env:TRAGIC}")
     result = replace_one("{env:MAGIC}")
     assert result == "{env:MAGIC}"
+    assert "circular chain between set env MAGIC, TRAGIC" in caplog.messages
+
+
+@pytest.mark.usefixtures("reset_env_var_after_delay")
+def test_replace_env_var_circular_flip_flop_5(
+    replace_one: ReplaceOne,
+    monkeypatch: MonkeyPatch,
+    caplog: LogCaptureFixture,
+) -> None:
+    """Replacement values will not infinitely loop back and forth (longer chain)"""
+    monkeypatch.setenv("MAGIC", "{env:TRAGIC}")
+    monkeypatch.setenv("TRAGIC", "{env:RABBIT}")
+    monkeypatch.setenv("RABBIT", "{env:HAT}")
+    monkeypatch.setenv("HAT", "{env:TRICK}")
+    monkeypatch.setenv("TRICK", "{env:MAGIC}")
+    result = replace_one("{env:MAGIC}")
+    assert result == "{env:MAGIC}"
+    assert "circular chain between set env MAGIC, TRAGIC, RABBIT, HAT, TRICK" in caplog.messages
 
 
 @pytest.mark.parametrize("fallback", [True, False])