Repository: pallets/jinja (python)

Issue:
Undefined objects can't be copied or pickled by Python > 3.5
### Description
Due to [an ancient erroneous deletion of `__slots__` metadata](https://github.com/pallets/jinja/blob/3.1.4/src/jinja2/runtime.py#L1049-L1056), it's not possible to copy/deepcopy/pickle `Undefined` objects (all of which we occasionally need to do in Ansible) in Python > 3.5.

The comment above the deletion of `__slots__` implies a misunderstanding of how that metadata was/is used. The deletion was always probably technically a bug, but once Python started providing default implementations of `__getstate__` and `__reduce_ex__` on all objects (which consult that metadata at runtime), it broke the default copyability/pickle-ability of `Undefined` (and derived) objects.

We're working around this with a monkeypatch that restores the missing `__slots__` metadata. We've also submitted PR  #2026 to zap the problematic deletion and tests that verify copy/deepcopy/pickle functionality on all the built-in `Undefined` types.

### Repro

(on any Python > 3.5 with Jinja installed):
```
$ python -c 'import copy; import jinja2; copy.copy(jinja2.Undefined())'
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/usr/lib64/python3.11/copy.py", line 92, in copy
    rv = reductor(4)
         ^^^^^^^^^^^
TypeError: cannot pickle 'Undefined' object
```

### Expected
The default runtime-provided implementations of `__getstate__` and `__reduce_ex__` that back `copy.copy`, `copy.deepcopy` and `pickle` should work properly with all `Undefined` derived types.

### Environment

- Python version: >= 3.6
- Jinja version: anything from at least the past 16 years


Pull request:
Preserve `__slots__` metadata on Undefined types
Restores erroneous deletions of `__slots__` metadata from `Undefined` types that breaks various serialization/copy mechanisms on Python >= 3.6. Also:

* fix ChainedUndefined to raise AttributeError on unimplemented dunder methods to prevent breaking core Python protocols (including copy)
* add tests

fixes #2025 


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index aebb38b5..58dc0321 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -22,6 +22,8 @@ Unreleased
     :issue:`1921`
 -   Make compiling deterministic for tuple unpacking in a ``{% set ... %}``
     call. :issue:`2021`
+-   Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined``
+    objects. :issue:`2025`
 
 
 Version 3.1.4
diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py
index c2c7c193..09119e2a 100644
--- a/src/jinja2/runtime.py
+++ b/src/jinja2/runtime.py
@@ -860,7 +860,11 @@ class Undefined:
 
     @internalcode
     def __getattr__(self, name: str) -> t.Any:
-        if name[:2] == "__":
+        # Raise AttributeError on requests for names that appear to be unimplemented
+        # dunder methods to keep Python's internal protocol probing behaviors working
+        # properly in cases where another exception type could cause unexpected or
+        # difficult-to-diagnose failures.
+        if name[:2] == "__" and name[-2:] == "__":
             raise AttributeError(name)
 
         return self._fail_with_undefined_error()
@@ -984,10 +988,20 @@ class ChainableUndefined(Undefined):
     def __html__(self) -> str:
         return str(self)
 
-    def __getattr__(self, _: str) -> "ChainableUndefined":
+    def __getattr__(self, name: str) -> "ChainableUndefined":
+        # Raise AttributeError on requests for names that appear to be unimplemented
+        # dunder methods to avoid confusing Python with truthy non-method objects that
+        # do not implement the protocol being probed for. e.g., copy.copy(Undefined())
+        # fails spectacularly if getattr(Undefined(), '__setstate__') returns an
+        # Undefined object instead of raising AttributeError to signal that it does not
+        # support that style of object initialization.
+        if name[:2] == "__" and name[-2:] == "__":
+            raise AttributeError(name)
+
         return self
 
-    __getitem__ = __getattr__  # type: ignore
+    def __getitem__(self, _name: str) -> "ChainableUndefined":  # type: ignore[override]
+        return self
 
 
 class DebugUndefined(Undefined):
@@ -1046,13 +1060,3 @@ class StrictUndefined(Undefined):
     __iter__ = __str__ = __len__ = Undefined._fail_with_undefined_error
     __eq__ = __ne__ = __bool__ = __hash__ = Undefined._fail_with_undefined_error
     __contains__ = Undefined._fail_with_undefined_error
-
-
-# Remove slots attributes, after the metaclass is applied they are
-# unneeded and contain wrong data for subclasses.
-del (
-    Undefined.__slots__,
-    ChainableUndefined.__slots__,
-    DebugUndefined.__slots__,
-    StrictUndefined.__slots__,
-)


Test patch:
diff --git a/tests/test_api.py b/tests/test_api.py
index ff3fcb13..ee11a8d6 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -323,8 +323,6 @@ class TestUndefined:
         assert und1 == und2
         assert und1 != 42
         assert hash(und1) == hash(und2) == hash(Undefined())
-        with pytest.raises(AttributeError):
-            getattr(Undefined, "__slots__")  # noqa: B009
 
     def test_chainable_undefined(self):
         env = Environment(undefined=ChainableUndefined)
@@ -335,8 +333,6 @@ class TestUndefined:
         assert env.from_string("{{ foo.missing }}").render(foo=42) == ""
         assert env.from_string("{{ not missing }}").render() == "True"
         pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render)
-        with pytest.raises(AttributeError):
-            getattr(ChainableUndefined, "__slots__")  # noqa: B009
 
         # The following tests ensure subclass functionality works as expected
         assert env.from_string('{{ missing.bar["baz"] }}').render() == ""
@@ -368,8 +364,6 @@ class TestUndefined:
             str(DebugUndefined(hint=undefined_hint))
             == f"{{{{ undefined value printed: {undefined_hint} }}}}"
         )
-        with pytest.raises(AttributeError):
-            getattr(DebugUndefined, "__slots__")  # noqa: B009
 
     def test_strict_undefined(self):
         env = Environment(undefined=StrictUndefined)
@@ -386,8 +380,6 @@ class TestUndefined:
             env.from_string('{{ missing|default("default", true) }}').render()
             == "default"
         )
-        with pytest.raises(AttributeError):
-            getattr(StrictUndefined, "__slots__")  # noqa: B009
         assert env.from_string('{{ "foo" if false }}').render() == ""
 
     def test_indexing_gives_undefined(self):
diff --git a/tests/test_runtime.py b/tests/test_runtime.py
index 1978c641..3cd3be15 100644
--- a/tests/test_runtime.py
+++ b/tests/test_runtime.py
@@ -1,6 +1,15 @@
+import copy
 import itertools
+import pickle
 
+import pytest
+
+from jinja2 import ChainableUndefined
+from jinja2 import DebugUndefined
+from jinja2 import StrictUndefined
 from jinja2 import Template
+from jinja2 import TemplateRuntimeError
+from jinja2 import Undefined
 from jinja2.runtime import LoopContext
 
 TEST_IDX_TEMPLATE_STR_1 = (
@@ -73,3 +82,44 @@ def test_mock_not_pass_arg_marker():
     out = t.render(calc=Calc())
     # Would be "1" if context argument was passed.
     assert out == "0"
+
+
+_undefined_types = (Undefined, ChainableUndefined, DebugUndefined, StrictUndefined)
+
+
+@pytest.mark.parametrize("undefined_type", _undefined_types)
+def test_undefined_copy(undefined_type):
+    undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError)
+    copied = copy.copy(undef)
+
+    assert copied is not undef
+    assert copied._undefined_hint is undef._undefined_hint
+    assert copied._undefined_obj is undef._undefined_obj
+    assert copied._undefined_name is undef._undefined_name
+    assert copied._undefined_exception is undef._undefined_exception
+
+
+@pytest.mark.parametrize("undefined_type", _undefined_types)
+def test_undefined_deepcopy(undefined_type):
+    undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError)
+    copied = copy.deepcopy(undef)
+
+    assert copied._undefined_hint is undef._undefined_hint
+    assert copied._undefined_obj is not undef._undefined_obj
+    assert copied._undefined_obj == undef._undefined_obj
+    assert copied._undefined_name is undef._undefined_name
+    assert copied._undefined_exception is undef._undefined_exception
+
+
+@pytest.mark.parametrize("undefined_type", _undefined_types)
+def test_undefined_pickle(undefined_type):
+    undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError)
+    copied = pickle.loads(pickle.dumps(undef))
+
+    assert copied._undefined_hint is not undef._undefined_hint
+    assert copied._undefined_hint == undef._undefined_hint
+    assert copied._undefined_obj is not undef._undefined_obj
+    assert copied._undefined_obj == undef._undefined_obj
+    assert copied._undefined_name is not undef._undefined_name
+    assert copied._undefined_name == undef._undefined_name
+    assert copied._undefined_exception is undef._undefined_exception