Repository: copier-org/copier (Python)

Issue:
UnicodeDecodeError when updating on Windows
### Describe the problem

During the update of a project to a new template version I got an UnicodeDecodeError.
The update was therefore incomplete.


### Template

Template: https://github.com/dalito/linkml-project-copier
Repo that was updated (from v0.2.2 to v0.3.0): https://github.com/nfdi4cat/pid4cat-model

### To Reproduce

_No response_

### Logs

```console
λ copier update --trust --skip-answered
Updating to template version 0.3.0
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "C:\Users\david\.local\bin\copier.exe\__main__.py", line 7, in <module>
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\plumbum\cli\application.py", line 640, in run
    inst, retcode = subapp.run(argv, exit=False)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\plumbum\cli\application.py", line 635, in run
    retcode = inst.main(*tailargs)
              ^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 425, in main
    return _handle_exceptions(inner)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 70, in _handle_exceptions
    method()
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 415, in inner
    with self._worker(
         ^^^^^^^^^^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 254, in __exit__
    raise value
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\cli.py", line 423, in inner
    worker.run_update()
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 1031, in run_update
    self._apply_update()
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 1239, in _apply_update
    if any(
       ^^^^
  File "C:\Users\david\.local\pipx\venvs\copier\Lib\site-packages\copier\main.py", line 1242, in <genexpr>
    for line in conflicts_candidate
                ^^^^^^^^^^^^^^^^^^^
  File "C:\dev\Python312\Lib\encodings\cp1252.py", line 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 712: character maps to <undefined>
```

### Expected behavior

The update should run to completion.

### Screenshots/screencasts/logs

_No response_

### Operating system

macOS

### Operating system distribution and version

Windows 10 Pro, German locale

### Copier version

copier 9.5.0

### Python version

CPython 3.12

### Installation method

pipx+pypi

### Additional context

_No response_


Discussion:

Patch:
diff --git a/copier/main.py b/copier/main.py
index f164ff073..625009c40 100644
--- a/copier/main.py
+++ b/copier/main.py
@@ -1232,10 +1232,13 @@ def _apply_update(self) -> None:  # noqa: C901
                         # The 3-way merge might have resolved conflicts automatically,
                         # so we need to check if the file contains conflict markers
                         # before storing the file name for marking it as unmerged after the loop.
-                        with Path(fname).open() as conflicts_candidate:
+                        with Path(fname).open("rb") as conflicts_candidate:
                             if any(
                                 line.rstrip()
-                                in {"<<<<<<< before updating", ">>>>>>> after updating"}
+                                in {
+                                    b"<<<<<<< before updating",
+                                    b">>>>>>> after updating",
+                                }
                                 for line in conflicts_candidate
                             ):
                                 conflicted.append(fname)


Test patch:
diff --git a/tests/helpers.py b/tests/helpers.py
index c50086b47..3e3e921fa 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -102,7 +102,9 @@ def assert_file(tmp_path: Path, *path: str) -> None:
 
 
 def build_file_tree(
-    spec: Mapping[StrOrPath, str | bytes | Path], dedent: bool = True
+    spec: Mapping[StrOrPath, str | bytes | Path],
+    dedent: bool = True,
+    encoding: str = "utf-8",
 ) -> None:
     """Builds a file tree based on the received spec.
 
@@ -124,7 +126,8 @@ def build_file_tree(
                 assert isinstance(contents, str)
                 contents = textwrap.dedent(contents)
             mode = "wb" if binary else "w"
-            with path.open(mode) as fd:
+            enc = None if binary else encoding
+            with Path(path).open(mode, encoding=enc) as fd:
                 fd.write(contents)
 
 
diff --git a/tests/test_exclude.py b/tests/test_exclude.py
index 516c4c965..db9d113ae 100644
--- a/tests/test_exclude.py
+++ b/tests/test_exclude.py
@@ -55,7 +55,7 @@ def test_config_include(tmp_path_factory: pytest.TempPathFactory) -> None:
 
 
 @pytest.mark.xfail(
-    condition=platform.system() in {"Darwin", "Windows"},
+    condition=platform.system() == "Darwin",
     reason="OS without proper UTF-8 filesystem.",
     strict=True,
 )
diff --git a/tests/test_updatediff.py b/tests/test_updatediff.py
index 35a7a45fb..07f42c3d8 100644
--- a/tests/test_updatediff.py
+++ b/tests/test_updatediff.py
@@ -1787,3 +1787,85 @@ def test_update_with_answers_with_umlaut(
     run_update(dst, skip_answered=True, overwrite=True)
     answers = load_answersfile_data(dst)
     assert answers["umlaut"] == "äöü"
+
+
+def test_conflict_on_update_with_unicode_in_content(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> None:
+    src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))
+    with local.cwd(src):
+        build_file_tree(
+            {
+                "{{ _copier_conf.answers_file }}.jinja": "{{ _copier_answers|to_yaml }}",
+                "copier.yml": "b: false",
+                "content.jinja": """\
+                    aaa🐍
+                    {%- if b %}
+                    bbb🐍
+                    {%- endif %}
+                    zzz🐍
+                    """,
+            },
+            encoding="utf-8",
+        )
+        git("init")
+        git("add", "-A")
+        git("commit", "-m1")
+        git("tag", "1")
+        build_file_tree(
+            {
+                "copier.yml": dedent(
+                    """\
+                    b: false
+                    c: false
+                    """
+                ),
+                "content.jinja": """\
+                    aaa🐍
+                    {%- if b %}
+                    bbb🐍
+                    {%- endif %}
+                    {%- if c %}
+                    ccc🐍
+                    {%- endif %}
+                    zzz🐍
+                    """,
+            },
+            encoding="utf-8",
+        )
+        git("commit", "-am2")
+        git("tag", "2")
+    # Init project
+    run_copy(str(src), dst, data={"b": True}, vcs_ref="1")
+    assert "ccc" not in (dst / "content").read_text(encoding="utf-8")
+    with local.cwd(dst):
+        git("init")
+        git("add", "-A")
+        git("commit", "-m1")
+        # Project evolution
+        Path("content").write_text(
+            dedent(
+                """\
+                aaa🐍
+                bbb🐍
+                jjj🐍
+                zzz🐍
+                """
+            ),
+            encoding="utf-8",
+        )
+        git("commit", "-am2")
+        # Update from template, inline, with answer changes
+        run_update(data={"c": True}, defaults=True, overwrite=True, conflict="inline")
+        assert Path("content").read_text(encoding="utf-8") == dedent(
+            """\
+            aaa🐍
+            bbb🐍
+            <<<<<<< before updating
+            jjj🐍
+            =======
+            ccc🐍
+            >>>>>>> after updating
+            zzz🐍
+            """
+        )