Repository: copier-org/copier (python)

Issue:
UnicodeDecodeError when updating on Windows
### Describe the problem

During the update of a project to a new template version I got an UnicodeDecodeError.
The update was therefore incomplete.


### Template

Template: https://github.com/dalito/linkml-project-copier
Repo that was updated (from v0.2.2 to v0.3.0): https://github.com/nfdi4cat/pid4cat-model

### To Reproduce

_No response_

### Logs

```console
λ copier update --trust --skip-answered
Updating to template version 0.3.0
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "C:\Users\david\.local\bin\copier.exe\__main__.py", line 7, in <module>
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\plumbum\cli\application.py", line 640, in run
    inst, retcode = subapp.run(argv, exit=False)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\plumbum\cli\application.py", line 635, in run
    retcode = inst.main(*tailargs)
              ^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 425, in main
    return _handle_exceptions(inner)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 70, in _handle_exceptions
    method()
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 415, in inner
    with self._worker(
         ^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 254, in __exit__
    raise value
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 423, in inner
    worker.run_update()
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 1031, in run_update
    self._apply_update()
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 1239, in _apply_update
    if any(
       ^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 1242, in <genexpr>
    for line in conflicts_candidate
                ^^^^^^^^^^^^^^^^^^^
  File "C:\dev\Python312\Lib\encodings\cp1252.py", line 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 712: character maps to <undefined>
```

### Expected behavior

The update should run to completion.

### Screenshots/screencasts/logs

_No response_

### Operating system

macOS

### Operating system distribution and version

Windows 10 Pro, German locale

### Copier version

copier 9.5.0

### Python version

CPython 3.12

### Installation method

pipx+pypi

### Additional context

_No response_

Pull request:
Explicitly specify file encoding for windows
It seems that this place was overlooked. The file encoding is specified on all other non-binary `open(...)` calls except for this one.

Closes #2006

Patch:
diff --git a/copier/main.py b/copier/main.py
index f164ff0..bb77f2f 100644
--- a/copier/main.py
+++ b/copier/main.py
@@ -7,9 +7,10 @@ import platform
 import subprocess
 import sys
 from contextlib import suppress
+from contextvars import ContextVar
 from dataclasses import asdict, field, replace
 from filecmp import dircmp
-from functools import cached_property, partial
+from functools import cached_property, partial, wraps
 from itertools import chain
 from pathlib import Path
 from shutil import rmtree
@@ -65,6 +66,8 @@ from .types import (
     AnyByStrMutableMapping,
     JSONSerializable,
     LazyDict,
+    Operation,
+    ParamSpec,
     Phase,
     RelativePath,
     StrOrPath,
@@ -73,6 +76,29 @@ from .user_data import AnswersMap, Question, load_answersfile_data
 from .vcs import get_git
 
 _T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+_operation: ContextVar[Operation] = ContextVar("_operation")
+
+
+def as_operation(value: Operation) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    """Decorator to set the current operation context, if not defined already.
+
+    This value is used to template specific configuration options.
+    """
+
+    def _decorator(func: Callable[_P, _T]) -> Callable[_P, _T]:
+        @wraps(func)
+        def _wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+            token = _operation.set(_operation.get(value))
+            try:
+                return func(*args, **kwargs)
+            finally:
+                _operation.reset(token)
+
+        return _wrapper
+
+    return _decorator
 
 
 @dataclass(config=ConfigDict(extra="forbid"))
@@ -248,7 +274,7 @@ class Worker:
         for method in self._cleanup_hooks:
             method()
 
-    def _check_unsafe(self, mode: Literal["copy", "update"]) -> None:
+    def _check_unsafe(self, mode: Operation) -> None:
         """Check whether a template uses unsafe features."""
         if self.unsafe or self.settings.is_trusted(self.template.url):
             return
@@ -327,8 +353,10 @@ class Worker:
         Arguments:
             tasks: The list of tasks to run.
         """
+        operation = _operation.get()
         for i, task in enumerate(tasks):
             extra_context = {f"_{k}": v for k, v in task.extra_vars.items()}
+            extra_context["_copier_operation"] = operation
 
             if not cast_to_bool(self._render_value(task.condition, extra_context)):
                 continue
@@ -358,7 +386,7 @@ class Worker:
                 / Path(self._render_string(str(task.working_directory), extra_context))
             ).absolute()
 
-            extra_env = {k.upper(): str(v) for k, v in task.extra_vars.items()}
+            extra_env = {k[1:].upper(): str(v) for k, v in extra_context.items()}
             with local.cwd(working_directory), local.env(**extra_env):
                 subprocess.run(task_cmd, shell=use_shell, check=True, env=local.env)
 
@@ -625,7 +653,14 @@ class Worker:
     @cached_property
     def match_exclude(self) -> Callable[[Path], bool]:
         """Get a callable to match paths against all exclusions."""
-        return self._path_matcher(self.all_exclusions)
+        # Include the current operation in the rendering context.
+        # Note: This method is a cached property, it needs to be regenerated
+        # when reusing an instance in different contexts.
+        extra_context = {"_copier_operation": _operation.get()}
+        return self._path_matcher(
+            self._render_string(exclusion, extra_context=extra_context)
+            for exclusion in self.all_exclusions
+        )
 
     @cached_property
     def match_skip(self) -> Callable[[Path], bool]:
@@ -928,6 +963,7 @@ class Worker:
         return self.template.local_abspath / subdir
 
     # Main operations
+    @as_operation("copy")
     def run_copy(self) -> None:
         """Generate a subproject from zero, ignoring what was in the folder.
 
@@ -938,6 +974,11 @@ class Worker:
 
         See [generating a project][generating-a-project].
         """
+        with suppress(AttributeError):
+            # We might have switched operation context, ensure the cached property
+            # is regenerated to re-render templates.
+            del self.match_exclude
+
         self._check_unsafe("copy")
         self._print_message(self.template.message_before_copy)
         with Phase.use(Phase.PROMPT):
@@ -967,6 +1008,7 @@ class Worker:
             # TODO Unify printing tools
             print("")  # padding space
 
+    @as_operation("copy")
     def run_recopy(self) -> None:
         """Update a subproject, keeping answers but discarding evolution."""
         if self.subproject.template is None:
@@ -977,6 +1019,7 @@ class Worker:
         with replace(self, src_path=self.subproject.template.url) as new_worker:
             new_worker.run_copy()
 
+    @as_operation("update")
     def run_update(self) -> None:
         """Update a subproject that was already generated.
 
@@ -1024,6 +1067,11 @@ class Worker:
             print(
                 f"Updating to template version {self.template.version}", file=sys.stderr
             )
+        with suppress(AttributeError):
+            # We might have switched operation context, ensure the cached property
+            # is regenerated to re-render templates.
+            del self.match_exclude
+
         self._apply_update()
         self._print_message(self.template.message_after_update)
 
@@ -1232,10 +1280,13 @@ class Worker:
                         # The 3-way merge might have resolved conflicts automatically,
                         # so we need to check if the file contains conflict markers
                         # before storing the file name for marking it as unmerged after the loop.
-                        with Path(fname).open() as conflicts_candidate:
+                        with Path(fname).open("rb") as conflicts_candidate:
                             if any(
                                 line.rstrip()
-                                in {"<<<<<<< before updating", ">>>>>>> after updating"}
+                                in {
+                                    b"<<<<<<< before updating",
+                                    b">>>>>>> after updating",
+                                }
                                 for line in conflicts_candidate
                             ):
                                 conflicted.append(fname)
diff --git a/copier/types.py b/copier/types.py
index a8fd982..bcfa57f 100644
--- a/copier/types.py
+++ b/copier/types.py
@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import sys
 from contextlib import contextmanager
 from contextvars import ContextVar
 from enum import Enum
@@ -24,6 +25,11 @@ from typing import (
 
 from pydantic import AfterValidator
 
+if sys.version_info >= (3, 10):
+    from typing import ParamSpec as ParamSpec
+else:
+    from typing_extensions import ParamSpec as ParamSpec
+
 # simple types
 StrOrPath = Union[str, Path]
 AnyByStrDict = Dict[str, Any]
@@ -44,6 +50,7 @@ VCSTypes = Literal["git"]
 Env = Mapping[str, str]
 MissingType = NewType("MissingType", object)
 MISSING = MissingType(object())
+Operation = Literal["copy", "update"]
 
 
 # Validators
diff --git a/docs/configuring.md b/docs/configuring.md
index 9d3c5e5..d40d60e 100644
--- a/docs/configuring.md
+++ b/docs/configuring.md
@@ -962,6 +962,21 @@ to know available options.
 
 The CLI option can be passed several times to add several patterns.
 
+Each pattern can be templated using Jinja.
+
+!!! example
+
+    Templating `exclude` patterns using `_copier_operation` allows to have files
+    that are rendered once during `copy`, but are never updated:
+
+    ```yaml
+    _exclude:
+        - "{% if _copier_operation == 'update' -%}src/*_example.py{% endif %}"
+    ```
+
+    The difference with [skip_if_exists][] is that it will never be rendered during
+    an update, no matter if it exitsts or not.
+
 !!! info
 
     When you define this parameter in `copier.yml`, it will **replace** the default
@@ -1421,6 +1436,8 @@ configuring `secret: true` in the [advanced prompt format][advanced-prompt-forma
 exist, but always be present. If they do not exist in a project during an `update`
 operation, they will be recreated.
 
+Each pattern can be templated using Jinja.
+
 !!! example
 
     For example, it can be used if your project generates a password the 1st time and
@@ -1571,6 +1588,9 @@ other items not present.
         - [invoke, end-process, "--full-conf={{ _copier_conf|to_json }}"]
         # Your script can be run by the same Python environment used to run Copier
         - ["{{ _copier_python }}", task.py]
+        # Run a command during the initial copy operation only, excluding updates
+        - command: ["{{ _copier_python }}", task.py]
+          when: "{{ _copier_operation == 'copy' }}"
         # OS-specific task (supported values are "linux", "macos", "windows" and `None`)
         - command: rm {{ name_of_the_project }}/README.md
           when: "{{ _copier_conf.os in  ['linux', 'macos'] }}"
diff --git a/docs/creating.md b/docs/creating.md
index 83e5398..b93129c 100644
--- a/docs/creating.md
+++ b/docs/creating.md
@@ -146,6 +146,16 @@ The current phase, one of `"prompt"`,`"tasks"`, `"migrate"` or `"render"`.
     You may encounter this phase when rendering outside of those phases,
     when rendering lazily (and the phase notion can be irrelevant) or when testing.
 
+## Variables (context-dependent)
+
+Some variables are only available in select contexts:
+
+### `_copier_operation`
+
+The current operation, either `"copy"` or `"update"`.
+
+Availability: [`exclude`](configuring.md#exclude), [`tasks`](configuring.md#tasks)
+
 ## Variables (context-specific)
 
 Some rendering contexts provide variables unique to them:
diff --git a/docs/generating.md b/docs/generating.md
index 1a4d042..b15bf58 100644
--- a/docs/generating.md
+++ b/docs/generating.md
@@ -18,6 +18,10 @@ Or within Python code:
 copier.run_copy("path/to/project/template", "path/to/destination")
 ```
 
+If `path/to/destination` doesn't exist, Copier will create the directory and populate it
+with the generated files. If `path/to/destination` exists, it must be writable (not
+read-only).
+
 The "template" parameter can be a local path, an URL, or a shortcut URL:
 
 -   GitHub: `gh:namespace/project`


Test patch:
diff --git a/tests/helpers.py b/tests/helpers.py
index c50086b..3e3e921 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -102,7 +102,9 @@ def assert_file(tmp_path: Path, *path: str) -> None:
 
 
 def build_file_tree(
-    spec: Mapping[StrOrPath, str | bytes | Path], dedent: bool = True
+    spec: Mapping[StrOrPath, str | bytes | Path],
+    dedent: bool = True,
+    encoding: str = "utf-8",
 ) -> None:
     """Builds a file tree based on the received spec.
 
@@ -124,7 +126,8 @@ def build_file_tree(
                 assert isinstance(contents, str)
                 contents = textwrap.dedent(contents)
             mode = "wb" if binary else "w"
-            with path.open(mode) as fd:
+            enc = None if binary else encoding
+            with Path(path).open(mode, encoding=enc) as fd:
                 fd.write(contents)
 
 
diff --git a/tests/test_context.py b/tests/test_context.py
new file mode 100644
index 0000000..9344ff4
--- /dev/null
+++ b/tests/test_context.py
@@ -0,0 +1,90 @@
+import json
+from pathlib import Path
+
+import pytest
+from plumbum import local
+
+import copier
+
+from .helpers import build_file_tree, git_save
+
+
+def test_exclude_templating_with_operation(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> None:
+    """
+    Ensure it's possible to create one-off boilerplate files that are not
+    managed during updates via `_exclude` using the `_copier_operation` context variable.
+    """
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+
+    template = "{% if _copier_operation == 'update' %}copy-only{% endif %}"
+    with local.cwd(src):
+        build_file_tree(
+            {
+                "copier.yml": f'_exclude:\n - "{template}"',
+                "{{ _copier_conf.answers_file }}.jinja": "{{ _copier_answers|to_yaml }}",
+                "copy-only": "foo",
+                "copy-and-update": "foo",
+            }
+        )
+        git_save(tag="1.0.0")
+        build_file_tree(
+            {
+                "copy-only": "bar",
+                "copy-and-update": "bar",
+            }
+        )
+        git_save(tag="2.0.0")
+    copy_only = dst / "copy-only"
+    copy_and_update = dst / "copy-and-update"
+
+    copier.run_copy(str(src), dst, defaults=True, overwrite=True, vcs_ref="1.0.0")
+    for file in (copy_only, copy_and_update):
+        assert file.exists()
+        assert file.read_text() == "foo"
+
+    with local.cwd(dst):
+        git_save()
+
+    copier.run_update(str(dst), overwrite=True)
+    assert copy_only.read_text() == "foo"
+    assert copy_and_update.read_text() == "bar"
+
+
+def test_task_templating_with_operation(
+    tmp_path_factory: pytest.TempPathFactory, tmp_path: Path
+) -> None:
+    """
+    Ensure that it is possible to define tasks that are only executed when copying.
+    """
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    # Use a file outside the Copier working directories to ensure accurate tracking
+    task_counter = tmp_path / "task_calls.txt"
+    with local.cwd(src):
+        build_file_tree(
+            {
+                "copier.yml": (
+                    f"""\
+                    _tasks:
+                        -   command: echo {{{{ _copier_operation }}}} >> {json.dumps(str(task_counter))}
+                            when: "{{{{ _copier_operation == 'copy' }}}}"
+                    """
+                ),
+                "{{ _copier_conf.answers_file }}.jinja": "{{ _copier_answers|to_yaml }}",
+            }
+        )
+        git_save(tag="1.0.0")
+
+    copier.run_copy(str(src), dst, defaults=True, overwrite=True, unsafe=True)
+    assert task_counter.exists()
+    assert len(task_counter.read_text().splitlines()) == 1
+
+    with local.cwd(dst):
+        git_save()
+
+    copier.run_recopy(dst, defaults=True, overwrite=True, unsafe=True)
+    assert len(task_counter.read_text().splitlines()) == 2
+
+    copier.run_update(dst, defaults=True, overwrite=True, unsafe=True)
+    assert len(task_counter.read_text().splitlines()) == 2
diff --git a/tests/test_exclude.py b/tests/test_exclude.py
index 516c4c9..db9d113 100644
--- a/tests/test_exclude.py
+++ b/tests/test_exclude.py
@@ -55,7 +55,7 @@ def test_config_include(tmp_path_factory: pytest.TempPathFactory) -> None:
 
 
 @pytest.mark.xfail(
-    condition=platform.system() in {"Darwin", "Windows"},
+    condition=platform.system() == "Darwin",
     reason="OS without proper UTF-8 filesystem.",
     strict=True,
 )
diff --git a/tests/test_updatediff.py b/tests/test_updatediff.py
index 35a7a45..07f42c3 100644
--- a/tests/test_updatediff.py
+++ b/tests/test_updatediff.py
@@ -1787,3 +1787,85 @@ def test_update_with_answers_with_umlaut(
     run_update(dst, skip_answered=True, overwrite=True)
     answers = load_answersfile_data(dst)
     assert answers["umlaut"] == "äöü"
+
+
+def test_conflict_on_update_with_unicode_in_content(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    with local.cwd(src):
+        build_file_tree(
+            {
+                "{{ _copier_conf.answers_file }}.jinja": "{{ _copier_answers|to_yaml }}",
+                "copier.yml": "b: false",
+                "content.jinja": """\
+                    aaa🐍
+                    {%- if b %}
+                    bbb🐍
+                    {%- endif %}
+                    zzz🐍
+                    """,
+            },
+            encoding="utf-8",
+        )
+        git("init")
+        git("add", "-A")
+        git("commit", "-m1")
+        git("tag", "1")
+        build_file_tree(
+            {
+                "copier.yml": dedent(
+                    """\
+                    b: false
+                    c: false
+                    """
+                ),
+                "content.jinja": """\
+                    aaa🐍
+                    {%- if b %}
+                    bbb🐍
+                    {%- endif %}
+                    {%- if c %}
+                    ccc🐍
+                    {%- endif %}
+                    zzz🐍
+                    """,
+            },
+            encoding="utf-8",
+        )
+        git("commit", "-am2")
+        git("tag", "2")
+    # Init project
+    run_copy(str(src), dst, data={"b": True}, vcs_ref="1")
+    assert "ccc" not in (dst / "content").read_text(encoding="utf-8")
+    with local.cwd(dst):
+        git("init")
+        git("add", "-A")
+        git("commit", "-m1")
+        # Project evolution
+        Path("content").write_text(
+            dedent(
+                """\
+                aaa🐍
+                bbb🐍
+                jjj🐍
+                zzz🐍
+                """
+            ),
+            encoding="utf-8",
+        )
+        git("commit", "-am2")
+        # Update from template, inline, with answer changes
+        run_update(data={"c": True}, defaults=True, overwrite=True, conflict="inline")
+        assert Path("content").read_text(encoding="utf-8") == dedent(
+            """\
+            aaa🐍
+            bbb🐍
+            <<<<<<< before updating
+            jjj🐍
+            =======
+            ccc🐍
+            >>>>>>> after updating
+            zzz🐍
+            """
+        )