Repository: copier-org/copier (python)

Issue:
Copier 9.5.0 fails to provide `data` in rendering context
### Describe the problem

I use copier programmatically to render a template. Some variables used in templates are asked from the user, while others are inserted by the code. The variables we want to insert into the template are passed in `data` to `copier.Worker`. Here's an example:


```python3
data = {'my_injected_variable': '17'}

with copier.Worker(
        src_path=str(tpl_dir),
        dst_path=dst_path,
        unsafe=True,
        data=data,
        quiet=True,
        defaults=False,
    ) as worker:
        worker.jinja_env.add_extension("my_package.jinja_ext.ContextUpdater")
        worker.run_copy()
```

I have a Jinja extension that extends this context even further (`my_package.jinja_ext.ContextUpdater`). It looks like this:

```python3
from copier_templates_extensions import ContextHook

class ContextUpdater(ContextHook):
    update = False

    def hook(self, context: dict):
        """Modify the template context."""
        from pprint import pprint
        pprint(context)
        assert "my_injected_variable" in context
        context["my_other_stuff"] = context["my_injected_variable"] + "something"
```

In copier 9.4, I get the following `context` and everything works just fine:

```python3
{'_copier_answers': {'_src_path': 'REMOVED'},
 '_copier_conf': {'answers': {'hidden': set(),
                              'init': {'my_injected_variable': '17'},
                              'last': {},
                              'metadata': {'_src_path': 'REMOVED'},
                              'user': {},
                              'user_defaults': {}},
                  'answers_file': PosixPath('.copier-answers.yml'),
                  'cleanup_on_error': True,
                  'conflict': 'inline',
                  'context_lines': 3,
                  'data': {'my_injected_variable': '17'},
                  'defaults': False,
                  'dst_path': PosixPath('REMOVED'),
                  'exclude': (),
                  'os': 'linux',
                  'overwrite': False,
                  'pretend': False,
                  'quiet': True,
                  'sep': '/',
                  'settings': Settings(defaults={}, trust=set()),
                  'skip_answered': False,
                  'skip_if_exists': (),
                  'skip_tasks': False,
                  'src_path': PosixPath('REMOVED'),
                  'unsafe': True,
                  'use_prereleases': False,
                  'user_defaults': {},
                  'vcs_ref': None,
                  'vcs_ref_hash': None},
 '_copier_python': '/usr/bin/python3.9',
 '_folder_name': 'ibm-systems-cte2-tools-spider',
 '_src_path': 'REMOVED',
 'cycler': <class 'jinja2.utils.Cycler'>,
 'dict': <class 'dict'>,
 'joiner': <class 'jinja2.utils.Joiner'>,
 'lipsum': <function generate_lorem_ipsum at 0x7f6460c55670>,
 'make_secret': <function _make_secret at 0x7f645cb9b3a0>,
 'namespace': <class 'jinja2.utils.Namespace'>,
 'now': <function _now at 0x7f645cabd280>,
 'pathjoin': <function Worker.jinja_env.<locals>._pathjoin at 0x7f6457efb820>,
 'range': <function safe_range at 0x7f645cd40ca0>,
 'my_injected_variable': '17'}
```

Note the top-level key `context['my_injected_variable']`!


In copier 9.5, I get the following `context`: 

```python3
{'_copier_answers': {'_src_path': 'REMOVED'},
 '_copier_conf': {'answers_file': PosixPath('.copier-answers.yml'),
                  'cleanup_on_error': True,
                  'conflict': 'inline',
                  'context_lines': 3,
                  'data': {'my_injected_variable': '17'},
                  'defaults': False,
                  'dst_path': PosixPath('REMOVED'),
                  'exclude': (),
                  'os': 'linux',
                  'overwrite': False,
                  'pretend': False,
                  'quiet': True,
                  'sep': '/',
                  'settings': Settings(defaults={}, trust=set()),
                  'skip_answered': False,
                  'skip_if_exists': (),
                  'skip_tasks': False,
                  'src_path': PosixPath('REMOVED'),
                  'unsafe': True,
                  'use_prereleases': False,
                  'user_defaults': {},
                  'vcs_ref': None,
                  'vcs_ref_hash': None},
 '_copier_python': '/usr/bin/python3.9',
 '_external_data': <copier.main._LazyDict object at 0x7f242e2bf130>,
 '_folder_name': 'ibm-systems-cte2-tools-spider',
 'cycler': <class 'jinja2.utils.Cycler'>,
 'dict': <class 'dict'>,
 'joiner': <class 'jinja2.utils.Joiner'>,
 'lipsum': <function generate_lorem_ipsum at 0x7f243b869550>,
 'make_secret': <function _make_secret at 0x7f24337953a0>,
 'namespace': <class 'jinja2.utils.Namespace'>,
 'now': <function _now at 0x7f24336b7040>,
 'pathjoin': <function Worker.jinja_env.<locals>._pathjoin at 0x7f2432b15940>,
 'range': <function safe_range at 0x7f2433937b80>}
```

Now the top-level key `context['my_injected_variable']` is missing, causing the problems I'm seeing.

This new new behavior got introduced in https://github.com/copier-org/copier/pull/1880.

It feels like a regression, but maybe I was just using the API wrongly before. What's the new way of achieving what I'm looking for?


### Operating system

Linux

### Operating system distribution and version

RHEL 8

### Copier version

9.5.0

### Python version

cpython 3.9

### Installation method

pipx+pypi

Pull request:
fix: expose only answers in question rendering context
I've reverted a change in #1880 that broke context manipulation via `copier-templates-extensions`. As part of #1880, we [extended the `copier.user_data.AnswersMap` to include additional Jinja context](https://github.com/copier-org/copier/pull/1880/files#diff-51a5fc56dc37de31d15f74ed1458469179fe72a8b697cbbc83b2a8e8e508f90bL455-R505) like `_copier_*` variables, which was called _system render context_. With this change, the Jinja context for rendering questions did not only include (previous) answers [but also system render context variables](https://github.com/copier-org/copier/pull/1880/files#diff-51a5fc56dc37de31d15f74ed1458469179fe72a8b697cbbc83b2a8e8e508f90bR510). Among those system render context variables is the [`answers_file` variable](https://github.com/copier-org/copier/pull/1880/files#diff-51a5fc56dc37de31d15f74ed1458469179fe72a8b697cbbc83b2a8e8e508f90bR385) whose value is the _rendered_ answers file path. When a Jinja context hook via the `copier-templates-extension` extension is registered, it intercepts each Jinja template render call. The first render call occurred when [passing the system render context to the `copier.user_data.AnswersMap` constructor](https://github.com/copier-org/copier/pull/1880/files#diff-51a5fc56dc37de31d15f74ed1458469179fe72a8b697cbbc83b2a8e8e508f90bR505) where the `answers_file` template was rendered. At this point, no answers – not even non-interactively provided data – had been added to the context yet. Thus, accessing such data in the context hook was failing.

Here is the stack trace that helped me debug this problem:

```
.venv/lib/python3.13/site-packages/copier/main.py:1334: in run_copy
    with Worker(src_path=src_path, dst_path=Path(dst_path), **kwargs) as worker:
.venv/lib/python3.13/site-packages/copier/main.py:254: in __exit__
    raise value
.venv/lib/python3.13/site-packages/copier/main.py:1335: in run_copy
    worker.run_copy()
.venv/lib/python3.13/site-packages/copier/main.py:950: in run_copy
    self._ask()
.venv/lib/python3.13/site-packages/copier/main.py:505: in _ask
    system=self._system_render_context(),
.venv/lib/python3.13/site-packages/copier/main.py:385: in _system_render_context
    "answers_file": self.answers_relpath,
.venv/lib/python3.13/site-packages/copier/main.py:582: in answers_relpath
    return Path(template.render(**self.answers.combined))
.venv/lib/python3.13/site-packages/jinja2/environment.py:1290: in render
    ctx = self.new_context(dict(*args, **kwargs))
.venv/lib/python3.13/site-packages/jinja2/environment.py:1388: in new_context
    return new_context(
.venv/lib/python3.13/site-packages/jinja2/runtime.py:117: in new_context
    return environment.context_class(
src/copier_templates_extensions/extensions/context.py:48: in __init__
    if "_copier_conf" in parent and (context := extension_self.hook(parent)) is not None:
```

I've solved the problem by removing the system render context from the `copier.user_data.AnswersMap` again and adding only external data instead. This way, all non-interactively provided data are part of the `copier.user_data.AnswersMap` before deriving the render context for the first render call.

Fixes #1977.

Patch:
diff --git a/copier/main.py b/copier/main.py
index 84dc32b..360f2b8 100644
--- a/copier/main.py
+++ b/copier/main.py
@@ -64,6 +64,7 @@ from .types import (
     AnyByStrDict,
     AnyByStrMutableMapping,
     JSONSerializable,
+    LazyDict,
     RelativePath,
     StrOrPath,
 )
@@ -73,20 +74,6 @@ from .vcs import get_git
 _T = TypeVar("_T")
 
 
-# HACK https://github.com/copier-org/copier/pull/1880#discussion_r1887491497
-class _LazyDict:
-    """A dict where values are functions that get evaluated only once when requested."""
-
-    def __init__(self, **kwargs: Callable[[], Any]):
-        self.pending = kwargs
-        self.done: dict[str, Any] = {}
-
-    def __getitem__(self, key: str) -> Any:
-        if key not in self.done:
-            self.done[key] = self.pending[key]()
-        return self.done[key]
-
-
 @dataclass(config=ConfigDict(extra="forbid"))
 class Worker:
     """Copier process state manager.
@@ -281,7 +268,7 @@ class Worker:
         if features:
             raise UnsafeTemplateError(sorted(features))
 
-    def _external_data(self) -> _LazyDict:
+    def _external_data(self) -> LazyDict:
         """Load external data lazily.
 
         Result keys are used for rendering, and values are the parsed contents
@@ -290,7 +277,7 @@ class Worker:
         Files will only be parsed lazily on 1st access. This helps avoiding
         circular dependencies when the file name also comes from a variable.
         """
-        return _LazyDict(
+        return LazyDict(
             **{
                 name: lambda path=path: load_answersfile_data(
                     self.dst_path, self._render_string(path)
@@ -300,8 +287,6 @@ class Worker:
         )
 
     def _print_message(self, message: str) -> None:
-        # On first use, at least we need the system render context
-        self.answers.system = self._system_render_context()
         if message and not self.quiet:
             print(self._render_string(message), file=sys.stderr)
 
@@ -368,13 +353,8 @@ class Worker:
             with local.cwd(working_directory), local.env(**extra_env):
                 subprocess.run(task_cmd, shell=use_shell, check=True, env=local.env)
 
-    def _system_render_context(self) -> AnyByStrMutableMapping:
-        """System reserved render context.
-
-        Most keys start with `_` because they're reserved.
-
-        Resolution of computed values is deferred until used for the 1st time.
-        """
+    def _render_context(self) -> AnyByStrMutableMapping:
+        """Produce render context for Jinja."""
         # Backwards compatibility
         # FIXME Remove it?
         conf = asdict(self)
@@ -390,9 +370,9 @@ class Worker:
             }
         )
         return dict(
+            **self.answers.combined,
             _copier_answers=self._answers_to_remember(),
             _copier_conf=conf,
-            _external_data=self._external_data(),
             _folder_name=self.subproject.local_abspath.name,
             _copier_python=sys.executable,
         )
@@ -502,7 +482,7 @@ class Worker:
             init=self.data,
             last=self.subproject.last_answers,
             metadata=self.template.metadata,
-            system=self._system_render_context(),
+            external=self._external_data(),
         )
 
         for var_name, details in self.template.questions_data.items():
@@ -564,8 +544,9 @@ class Worker:
                     self.answers, question, self.template
                 ) from err
             self.answers.user[var_name] = new_answer
-        # Update system render context, which may depend on answers
-        self.answers.system = self._system_render_context()
+
+        # Reload external data, which may depend on answers
+        self.answers.external = self._external_data()
 
     @property
     def answers_relpath(self) -> Path:
@@ -696,7 +677,7 @@ class Worker:
                 new_content = src_abspath.read_bytes()
             else:
                 new_content = tpl.render(
-                    **self.answers.combined, **(extra_context or {})
+                    **self._render_context(), **(extra_context or {})
                 ).encode()
                 if self.jinja_env.yield_name:
                     raise YieldTagInFileError(
@@ -882,7 +863,7 @@ class Worker:
                 Additional variables to use for rendering the template.
         """
         tpl = self.jinja_env.from_string(string)
-        return tpl.render(**self.answers.combined, **(extra_context or {}))
+        return tpl.render(**self._render_context(), **(extra_context or {}))
 
     def _render_value(
         self, value: _T, extra_context: AnyByStrDict | None = None
@@ -1103,7 +1084,7 @@ class Worker:
                 )
             # Clear last answers cache to load possible answers migration, if skip_answered flag is not set
             if self.skip_answered is False:
-                self.answers = AnswersMap(system=self._system_render_context())
+                self.answers = AnswersMap(external=self._external_data())
                 with suppress(AttributeError):
                     del self.subproject.last_answers
             # Do a normal update in final destination
@@ -1119,7 +1100,7 @@ class Worker:
             ) as current_worker:
                 current_worker.run_copy()
                 self.answers = current_worker.answers
-                self.answers.system = self._system_render_context()
+                self.answers.external = self._external_data()
             # Render with the same answers in an empty dir to avoid pollution
             with replace(
                 self,
diff --git a/copier/types.py b/copier/types.py
index 740cf7d..f7ca137 100644
--- a/copier/types.py
+++ b/copier/types.py
@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import (
     Annotated,
     Any,
+    Callable,
     Dict,
     Literal,
     Mapping,
@@ -60,3 +61,17 @@ def path_is_relative(value: Path) -> Path:
 
 AbsolutePath = Annotated[Path, AfterValidator(path_is_absolute)]
 RelativePath = Annotated[Path, AfterValidator(path_is_relative)]
+
+
+# HACK https://github.com/copier-org/copier/pull/1880#discussion_r1887491497
+class LazyDict:
+    """A dict where values are functions that get evaluated only once when requested."""
+
+    def __init__(self, **kwargs: Callable[[], Any]):
+        self.pending = kwargs
+        self.done: dict[str, Any] = {}
+
+    def __getitem__(self, key: str) -> Any:
+        if key not in self.done:
+            self.done[key] = self.pending[key]()
+        return self.done[key]
diff --git a/copier/user_data.py b/copier/user_data.py
index 429a3e7..eec2cfc 100644
--- a/copier/user_data.py
+++ b/copier/user_data.py
@@ -27,7 +27,14 @@ from copier.settings import Settings
 
 from .errors import InvalidTypeError, MissingFileWarning, UserMessageError
 from .tools import cast_to_bool, cast_to_str, force_str_end
-from .types import MISSING, AnyByStrDict, AnyByStrMutableMapping, MissingType, StrOrPath
+from .types import (
+    MISSING,
+    AnyByStrDict,
+    AnyByStrMutableMapping,
+    LazyDict,
+    MissingType,
+    StrOrPath,
+)
 
 
 # TODO Remove these two functions as well as DEFAULT_DATA in a future release
@@ -57,7 +64,7 @@ DEFAULT_DATA: AnyByStrDict = {
 }
 
 
-@dataclass
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class AnswersMap:
     """Object that gathers answers from different sources.
 
@@ -99,7 +106,7 @@ class AnswersMap:
     metadata: AnyByStrMutableMapping = field(default_factory=dict)
     last: AnyByStrMutableMapping = field(default_factory=dict)
     user_defaults: AnyByStrMutableMapping = field(default_factory=dict)
-    system: AnyByStrMutableMapping = field(default_factory=dict)
+    external: LazyDict = field(default_factory=LazyDict)
 
     @property
     def combined(self) -> Mapping[str, Any]:
@@ -111,7 +118,7 @@ class AnswersMap:
                 self.metadata,
                 self.last,
                 self.user_defaults,
-                self.system,
+                {"_external_data": self.external},
                 DEFAULT_DATA,
             )
         )


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index 53b2644..448ec8e 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -346,7 +346,7 @@ def is_subdict(small: dict[Any, Any], big: dict[Any, Any]) -> bool:
 def test_worker_good_data(tmp_path: Path) -> None:
     # This test is probably useless, as it tests the what and not the how
     conf = copier.Worker("./tests/demo_data", tmp_path)
-    assert conf._system_render_context()["_folder_name"] == tmp_path.name
+    assert conf._render_context()["_folder_name"] == tmp_path.name
     assert conf.all_exclusions == ("exclude1", "exclude2")
     assert conf.template.skip_if_exists == ["skip_if_exists1", "skip_if_exists2"]
     assert conf.template.tasks == [