Repository: jazzband/django-debug-toolbar

Issue:
IndexError: list index out of range - when a repr renders templates
Related to: https://github.com/wagtail/wagtail/issues/5243

The `__repr__` method on an object in template context rendered a form, instead of just a simple string. Newer versions of Django have forms with widgets which render templates in them.

For some reason, with Postgres, a ModelChoiceIterator on a form, with a repr method which starts rendering templates, calling `pformat` on those variables in template context immediately evaluates them.

I've got a test case for it, which will be in the PR which hopefully fixes it.

```
Traceback (most recent call last):
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/core/handlers/exception.py", line 34, in inner
    response = get_response(request)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/core/handlers/base.py", line 115, in _get_response
    response = self.process_exception_by_middleware(e, request)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/core/handlers/base.py", line 113, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/views/decorators/cache.py", line 44, in _wrapped_view_func
    response = view_func(request, *args, **kwargs)
  File "/Users/tomkins/Python/wagtail/wagtail/admin/urls/__init__.py", line 102, in wrapper
    return view_func(request, *args, **kwargs)
  File "/Users/tomkins/Python/wagtail/wagtail/admin/decorators.py", line 34, in decorated_view
    return view_func(request, *args, **kwargs)
  File "/Users/tomkins/Python/wagtail/wagtail/admin/views/pages.py", line 539, in edit
    'has_unsaved_changes': has_unsaved_changes,
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/shortcuts.py", line 36, in render
    content = loader.render_to_string(template_name, context, request, using=using)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/loader.py", line 62, in render_to_string
    return template.render(context, request)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/backends/django.py", line 61, in render
    return self.template.render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/base.py", line 171, in render
    return self._render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/test/utils.py", line 95, in instrumented_test_render
    template_rendered.send(sender=self, template=self, context=context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/dispatch/dispatcher.py", line 175, in send
    for receiver in self._live_receivers(sender)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/dispatch/dispatcher.py", line 175, in <listcomp>
    for receiver in self._live_receivers(sender)
  File "/Users/tomkins/Python/django-debug-toolbar/debug_toolbar/panels/templates/panel.py", line 150, in _store_template_info
    pformatted = pformat(temp_layer)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 58, in pformat
    compact=compact).pformat(object)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 144, in pformat
    self._format(object, sio, 0, 0, {}, 0)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 161, in _format
    rep = self._repr(object, context, level)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 393, in _repr
    self._depth, level)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 405, in format
    return _safe_repr(object, context, maxlevels, level)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 514, in _safe_repr
    vrepr, vreadable, vrecur = saferepr(v, context, maxlevels, level)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pprint.py", line 555, in _safe_repr
    rep = repr(object)
  File "/Users/tomkins/Python/wagtail/wagtail/admin/edit_handlers.py", line 188, in __repr__
    self.model, self.instance, self.request, self.form)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/utils/html.py", line 388, in <lambda>
    klass.__str__ = lambda self: mark_safe(klass_str(self))
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/forms/forms.py", line 142, in __str__
    return self.as_table()
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/forms/forms.py", line 284, in as_table
    errors_on_separate_row=False,
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/forms/forms.py", line 243, in _html_output
    'field_name': bf.html_name,
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/utils/html.py", line 388, in <lambda>
    klass.__str__ = lambda self: mark_safe(klass_str(self))
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/forms/boundfield.py", line 33, in __str__
    return self.as_widget()
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/forms/boundfield.py", line 93, in as_widget
    renderer=self.form.renderer,
  File "/Users/tomkins/Python/wagtail/wagtail/core/blocks/base.py", line 505, in render
    return self.render_with_errors(name, value, attrs=attrs, errors=None, renderer=renderer)
  File "/Users/tomkins/Python/wagtail/wagtail/core/blocks/base.py", line 502, in render_with_errors
    return mark_safe(bound_block.render_form() + js_snippet)
  File "/Users/tomkins/Python/wagtail/wagtail/core/blocks/base.py", line 417, in render_form
    return self.block.render_form(self.value, self.prefix, errors=self.errors)
  File "/Users/tomkins/Python/wagtail/wagtail/core/blocks/stream_block.py", line 141, in render_form
    for (i, child) in enumerate(valid_children)
  File "/Users/tomkins/Python/wagtail/wagtail/core/blocks/stream_block.py", line 141, in <listcomp>
    for (i, child) in enumerate(valid_children)
  File "/Users/tomkins/Python/wagtail/wagtail/core/blocks/stream_block.py", line 78, in render_list_member
    'block_id': id,
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/loader.py", line 62, in render_to_string
    return template.render(context, request)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/backends/django.py", line 61, in render
    return self.template.render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/base.py", line 171, in render
    return self._render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/test/utils.py", line 96, in instrumented_test_render
    return self.nodelist.render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/base.py", line 937, in render
    bit = node.render_annotated(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/base.py", line 904, in render_annotated
    return self.render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/template/loader_tags.py", line 150, in render
    return compiled_parent._render(context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/test/utils.py", line 95, in instrumented_test_render
    template_rendered.send(sender=self, template=self, context=context)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/dispatch/dispatcher.py", line 175, in send
    for receiver in self._live_receivers(sender)
  File "/Users/tomkins/.virtualenvs/devsoc/lib/python3.6/site-packages/django/dispatch/dispatcher.py", line 175, in <listcomp>
    for receiver in self._live_receivers(sender)
  File "/Users/tomkins/Python/django-debug-toolbar/debug_toolbar/panels/templates/panel.py", line 103, in _store_template_info
    pformatted = self.pformat_layers[index]
IndexError: list index out of range
Unable to get template source from exception
```


Discussion:

Resolving patch:
diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py
--- a/debug_toolbar/panels/templates/panel.py
+++ b/debug_toolbar/panels/templates/panel.py
@@ -141,13 +141,16 @@ def _store_template_info(self, sender, **kwargs):
                                 temp_layer[key] = value
                             finally:
                                 recording(True)
+                    # Execute pformat first - if for some reason pformat/repr
+                    # causes more templates to be rendered, seen/pformat layers
+                    # will still be consistent
+                    pformatted = pformat(temp_layer)
                     # Refs GitHub issue #910
                     # If we've not seen the layer before then we will add it
                     # so that if we see it again we can skip formatting it.
                     self.seen_layers.append(key_values)
                     # Note: this *ought* to be len(...) - 1 but let's be safe.
                     index = self.seen_layers.index(key_values)
-                    pformatted = pformat(temp_layer)
                     # Note: this *ought* to be len(...) - 1 but let's be safe.
                     self.pformat_layers.insert(index, pformatted)
                     context_list.append(pformatted)


Test patch:
diff --git a/tests/forms.py b/tests/forms.py
new file mode 100644
--- /dev/null
+++ b/tests/forms.py
@@ -0,0 +1,9 @@
+from django import forms
+from django.contrib.auth.models import User
+
+
+class TemplateReprForm(forms.Form):
+    user = forms.ModelChoiceField(queryset=User.objects.all())
+
+    def __repr__(self):
+        return str(self)
diff --git a/tests/panels/test_template.py b/tests/panels/test_template.py
--- a/tests/panels/test_template.py
+++ b/tests/panels/test_template.py
@@ -3,6 +3,7 @@
 from django.test import TestCase, override_settings
 
 from ..base import BaseTestCase
+from ..forms import TemplateReprForm
 from ..models import NonAsciiRepr
 
 
@@ -35,6 +36,18 @@ def test_queryset_hook(self):
         self.assertIn("<<queryset of auth.User>>", ctx)
         self.assertIn("<<triggers database query>>", ctx)
 
+    def test_template_repr(self):
+        # Force widget templates to be included
+        self.toolbar.config["SKIP_TEMPLATE_PREFIXES"] = ()
+
+        User.objects.create(username="admin")
+        bad_repr = TemplateReprForm()
+        t = Template("{{ bad_repr }}")
+        c = Context({"bad_repr": bad_repr})
+        html = t.render(c)
+        self.assertIsNotNone(html)
+        self.assertValidHTML(html)
+
     def test_object_with_non_ascii_repr_in_context(self):
         response = self.panel.process_request(self.request)
         t = Template("{{ object }}")