Repository: pallets/jinja
Pull request #1164 (2020-03-06 13:20:49)

Don't use setlocal in syntax file
I'm not 100% sure that one should not use `setlocal` in a syntax file, but this breaks other filetypes that may include this syntax file, like the vim-markdown plugin if `jinja` is included in its `g:markdown_fenced_languages`.  The `comments` and `commentstring` end up set in the markdown buffer with the jinja settings.

See https://github.com/dhruvasagar/vim-table-mode/issues/162#issuecomment-595317626 for an example of side effects.


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index e7542118..1911ab40 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -9,6 +9,8 @@ Unreleased
 -   Bump MarkupSafe dependency to >=1.1.
 -   Bump Babel optional dependency to >=2.1.
 -   Remove code that was marked deprecated.
+-   Use :pep:`451` API to load templates with
+    :class:`~loaders.PackageLoader`. :issue:`1168`
 
 
 2.11.2
@@ -21,6 +23,22 @@ Unreleased
     :func:`contextfunction`. :issue:`1145`
 -   Update ``wordcount`` filter to trigger :class:`Undefined` methods
     by wrapping the input in :func:`soft_str`. :pr:`1160`
+-   Fix a hang when displaying tracebacks on Python 32-bit.
+    :issue:`1162`
+-   Showing an undefined error for an object that raises
+    ``AttributeError`` on access doesn't cause a recursion error.
+    :issue:`1177`
+-   Revert changes to :class:`~loaders.PackageLoader` from 2.10 which
+    removed the dependency on setuptools and pkg_resources, and added
+    limited support for namespace packages. The changes caused issues
+    when using Pytest. Due to the difficulty in supporting Python 2 and
+    :pep:`451` simultaneously, the changes are reverted until 3.0.
+    :pr:`1182`
+-   Fix line numbers in error messages when newlines are stripped.
+    :pr:`1178`
+-   The special ``namespace()`` assignment object in templates works in
+    async environments. :issue:`1180`
+
 
 Version 2.11.1
 --------------
diff --git a/docs/api.rst b/docs/api.rst
index c92c383e..53cb03b5 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -5,9 +5,10 @@ API
     :noindex:
     :synopsis: public Jinja API
 
-This document describes the API to Jinja and not the template language.  It
-will be most useful as reference to those implementing the template interface
-to the application and not those who are creating Jinja templates.
+This document describes the API to Jinja and not the template language
+(for that, see :doc:`/templates`). It will be most useful as reference
+to those implementing the template interface to the application and not
+those who are creating Jinja templates.
 
 Basics
 ------
diff --git a/docs/templates.rst b/docs/templates.rst
index c08ce4b4..a346ef23 100644
--- a/docs/templates.rst
+++ b/docs/templates.rst
@@ -912,9 +912,9 @@ Here's an example of how a call block can be used with arguments::
 
     {% call(user) dump_users(list_of_user) %}
         <dl>
-            <dl>Realname</dl>
+            <dt>Realname</dt>
             <dd>{{ user.realname|e }}</dd>
-            <dl>Description</dl>
+            <dt>Description</dt>
             <dd>{{ user.description }}</dd>
         </dl>
     {% endcall %}
diff --git a/ext/Vim/jinja.vim b/ext/Vim/jinja.vim
index e2a5bbf6..63f0d99a 100644
--- a/ext/Vim/jinja.vim
+++ b/ext/Vim/jinja.vim
@@ -82,9 +82,6 @@ syn region jinjaRaw matchgroup=jinjaRawDelim start="{%\s*raw\s*%}" end="{%\s*end
 
 " Jinja comments
 syn region jinjaComment matchgroup=jinjaCommentDelim start="{#" end="#}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString,jinjaComment
-" help support folding for some setups
-setlocal commentstring={#%s#}
-setlocal comments=s:{#,e:#}
 
 " Block start keywords.  A bit tricker.  We only highlight at the start of a
 " tag block and only if the name is not followed by a comma or equals sign
diff --git a/src/jinja2/debug.py b/src/jinja2/debug.py
index 46c24eba..5cac28ba 100644
--- a/src/jinja2/debug.py
+++ b/src/jinja2/debug.py
@@ -238,10 +238,7 @@ else:
     class _CTraceback(ctypes.Structure):
         _fields_ = [
             # Extra PyObject slots when compiled with Py_TRACE_REFS.
-            (
-                "PyObject_HEAD",
-                ctypes.c_byte * (32 if hasattr(sys, "getobjects") else 16),
-            ),
+            ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()),
             # Only care about tb_next as an object, not a traceback.
             ("tb_next", ctypes.py_object),
         ]
diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py
index e0b7a2e9..fc44dbeb 100644
--- a/src/jinja2/lexer.py
+++ b/src/jinja2/lexer.py
@@ -636,6 +636,7 @@ class Lexer:
         source_length = len(source)
         balancing_stack = []
         lstrip_unless_re = self.lstrip_unless_re
+        newlines_stripped = 0
 
         while 1:
             # tokenizer loop
@@ -672,7 +673,9 @@ class Lexer:
 
                         if strip_sign == "-":
                             # Strip all whitespace between the text and the tag.
-                            groups = (text.rstrip(),) + groups[1:]
+                            stripped = text.rstrip()
+                            newlines_stripped = text[len(stripped) :].count("\n")
+                            groups = (stripped,) + groups[1:]
                         elif (
                             # Not marked for preserving whitespace.
                             strip_sign != "+"
@@ -712,7 +715,8 @@ class Lexer:
                             data = groups[idx]
                             if data or token not in ignore_if_empty:
                                 yield lineno, token, data
-                            lineno += data.count("\n")
+                            lineno += data.count("\n") + newlines_stripped
+                            newlines_stripped = 0
 
                 # strings as token just are yielded as it.
                 else:
diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py
index b02cc357..09f678d7 100644
--- a/src/jinja2/loaders.py
+++ b/src/jinja2/loaders.py
@@ -1,10 +1,11 @@
 """API and implementations for loading templates from different data
 sources.
 """
+import importlib.util
 import os
-import pkgutil
 import sys
 import weakref
+import zipimport
 from collections import abc
 from hashlib import sha1
 from importlib import import_module
@@ -54,8 +55,8 @@ class BaseLoader:
                 if not exists(path):
                     raise TemplateNotFound(template)
                 mtime = getmtime(path)
-                with file(path) as f:
-                    source = f.read().decode('utf-8')
+                with open(path) as f:
+                    source = f.read()
                 return source, path, lambda: mtime == getmtime(path)
     """
 
@@ -232,10 +233,10 @@ class PackageLoader(BaseLoader):
     contributor. Zip files contributing to a namespace are not
     supported.
 
-    .. versionchanged:: 2.11.0
+    .. versionchanged:: 3.0
         No longer uses ``setuptools`` as a dependency.
 
-    .. versionchanged:: 2.11.0
+    .. versionchanged:: 3.0
         Limited PEP 420 namespace package support.
     """
 
@@ -253,21 +254,19 @@ class PackageLoader(BaseLoader):
         # Make sure the package exists. This also makes namespace
         # packages work, otherwise get_loader returns None.
         import_module(package_name)
-        self._loader = loader = pkgutil.get_loader(package_name)
-
-        # Zip loader's archive attribute points at the zip.
-        self._archive = getattr(loader, "archive", None)
+        spec = importlib.util.find_spec(package_name)
+        self._loader = loader = spec.loader
+        self._archive = None
         self._template_root = None
 
-        if hasattr(loader, "get_filename"):
-            # A standard directory package, or a zip package.
-            self._template_root = os.path.join(
-                os.path.dirname(loader.get_filename(package_name)), package_path
-            )
-        elif hasattr(loader, "_path"):
-            # A namespace package, limited support. Find the first
-            # contributor with the template directory.
-            for root in loader._path:
+        if isinstance(loader, zipimport.zipimporter):
+            self._archive = loader.archive
+            pkgdir = next(iter(spec.submodule_search_locations))
+            self._template_root = os.path.join(pkgdir, package_path)
+        elif spec.submodule_search_locations:
+            # This will be one element for regular packages and multiple
+            # for namespace packages.
+            for root in spec.submodule_search_locations:
                 root = os.path.join(root, package_path)
 
                 if os.path.isdir(root):
diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py
index 99293cd6..8ee02958 100644
--- a/src/jinja2/utils.py
+++ b/src/jinja2/utils.py
@@ -158,12 +158,13 @@ def object_type_repr(obj):
         return "None"
     elif obj is Ellipsis:
         return "Ellipsis"
-    # __builtin__ in 2.x, builtins in 3.x
-    if obj.__class__.__module__ in ("__builtin__", "builtins"):
-        name = obj.__class__.__name__
-    else:
-        name = f"{obj.__class__.__module__}.{obj.__class__.__name__}"
-    return f"{name} object"
+
+    cls = type(obj)
+
+    if cls.__module__ == "builtins":
+        return f"{cls.__name__} object"
+
+    return f"{cls.__module__}.{cls.__name__} object"
 
 
 def pformat(obj):
@@ -642,7 +643,8 @@ class Namespace:
         self.__attrs = dict(*args, **kwargs)
 
     def __getattribute__(self, name):
-        if name == "_Namespace__attrs":
+        # __class__ is needed for the awaitable check in async mode
+        if name in {"_Namespace__attrs", "__class__"}:
             return object.__getattribute__(self, name)
         try:
             return self.__attrs[name]


Test patch:
diff --git a/tests/test_api.py b/tests/test_api.py
index 0655c479..2679e8fb 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -267,6 +267,21 @@ class TestUndefined:
         with pytest.raises(AttributeError):
             Undefined("Foo").__dict__
 
+    def test_undefined_attribute_error(self):
+        # Django's LazyObject turns the __class__ attribute into a
+        # property that resolves the wrapped function. If that wrapped
+        # function raises an AttributeError, printing the repr of the
+        # object in the undefined message would cause a RecursionError.
+        class Error:
+            @property
+            def __class__(self):
+                raise AttributeError()
+
+        u = Undefined(obj=Error(), name="hello")
+
+        with pytest.raises(UndefinedError):
+            getattr(u, "recursion", None)
+
     def test_logging_undefined(self):
         _messages = []
 
diff --git a/tests/test_async.py b/tests/test_async.py
index 45713d5d..bfdcdb21 100644
--- a/tests/test_async.py
+++ b/tests/test_async.py
@@ -577,3 +577,14 @@ class TestAsyncForLoop:
     def test_awaitable_property_slicing(self, test_env_async):
         t = test_env_async.from_string("{% for x in a.b[:1] %}{{ x }}{% endfor %}")
         assert t.render(a=dict(b=[1, 2, 3])) == "1"
+
+
+def test_namespace_awaitable(test_env_async):
+    async def _test():
+        t = test_env_async.from_string(
+            '{% set ns = namespace(foo="Bar") %}{{ ns.foo }}'
+        )
+        actual = await t.render_async()
+        assert actual == "Bar"
+
+    run(_test())
diff --git a/tests/test_lexnparse.py b/tests/test_lexnparse.py
index d14a6ccb..60574169 100644
--- a/tests/test_lexnparse.py
+++ b/tests/test_lexnparse.py
@@ -163,6 +163,24 @@ class TestLexer:
         else:
             pytest.raises(TemplateSyntaxError, env.from_string, t)
 
+    def test_lineno_with_strip(self, env):
+        tokens = env.lex(
+            """\
+<html>
+    <body>
+    {%- block content -%}
+        <hr>
+        {{ item }}
+    {% endblock %}
+    </body>
+</html>"""
+        )
+        for tok in tokens:
+            lineno, token_type, value = tok
+            if token_type == "name" and value == "item":
+                assert lineno == 5
+                break
+
 
 class TestParser:
     def test_php_syntax(self, env):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 1e081334..8ca1289a 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,3 +1,6 @@
+import importlib.abc
+import importlib.machinery
+import importlib.util
 import os
 import platform
 import shutil
@@ -347,3 +350,34 @@ def test_package_zip_source(package_zip_loader, template, expect):
 )
 def test_package_zip_list(package_zip_loader):
     assert package_zip_loader.list_templates() == ["foo/test.html", "test.html"]
+
+
+def test_pep_451_import_hook():
+    class ImportHook(importlib.abc.MetaPathFinder, importlib.abc.Loader):
+        def find_spec(self, name, path=None, target=None):
+            if name != "res":
+                return None
+
+            spec = importlib.machinery.PathFinder.find_spec(name)
+            return importlib.util.spec_from_file_location(
+                name,
+                spec.origin,
+                loader=self,
+                submodule_search_locations=spec.submodule_search_locations,
+            )
+
+        def create_module(self, spec):
+            return None  # default behaviour is fine
+
+        def exec_module(self, module):
+            return None  # we need this to satisfy the interface, it's wrong
+
+    # ensure we restore `sys.meta_path` after putting in our loader
+    before = sys.meta_path[:]
+
+    try:
+        sys.meta_path.insert(0, ImportHook())
+        package_loader = PackageLoader("res")
+        assert "test.html" in package_loader.list_templates()
+    finally:
+        sys.meta_path[:] = before