Repository: ckan/ckan

Issue:
DataStore status page throws TypeError - Bleach upgrade regression
### CKAN Version if known (or site URL)
Git master (2ba282973690f481b1ed075ed9ce54db3f30ba5d)

### Please describe the expected behaviour
DataStore status page displays details of DataPusher processing.

### Please describe the actual behaviour
DataStore status page fails with Error 500 and following traceback in CKAN log
```
File '/usr/lib/python2.7/site-packages/weberror/errormiddleware.py', line 171 in __call__
  app_iter = self.application(environ, sr_checker)
File '/usr/lib/python2.7/site-packages/webob/dec.py', line 147 in __call__
  resp = self.call_func(req, *args, **self.kwargs)
File '/usr/lib/python2.7/site-packages/webob/dec.py', line 208 in call_func
  return self.func(req, *args, **kwargs)
File '/usr/lib/python2.7/site-packages/fanstatic/publisher.py', line 234 in __call__
  return request.get_response(self.app)
File '/usr/lib/python2.7/site-packages/webob/request.py', line 1053 in get_response
  application, catch_exc_info=False)
File '/usr/lib/python2.7/site-packages/webob/request.py', line 1022 in call_application
  app_iter = application(self.environ, start_response)
File '/usr/lib/python2.7/site-packages/webob/dec.py', line 147 in __call__
  resp = self.call_func(req, *args, **self.kwargs)
File '/usr/lib/python2.7/site-packages/webob/dec.py', line 208 in call_func
  return self.func(req, *args, **kwargs)
File '/usr/lib/python2.7/site-packages/fanstatic/injector.py', line 54 in __call__
  response = request.get_response(self.app)
File '/usr/lib/python2.7/site-packages/webob/request.py', line 1053 in get_response
  application, catch_exc_info=False)
File '/usr/lib/python2.7/site-packages/webob/request.py', line 1022 in call_application
  app_iter = application(self.environ, start_response)
File '/srv/ckan/src/ckan/ckan/config/middleware/pylons_app.py', line 266 in inner
  result = application(environ, start_response)
File '/usr/lib/python2.7/site-packages/beaker/middleware.py', line 73 in __call__
  return self.app(environ, start_response)
File '/usr/lib/python2.7/site-packages/beaker/middleware.py', line 156 in __call__
  return self.wrap_app(environ, session_start_response)
File '/usr/lib/python2.7/site-packages/routes/middleware.py', line 131 in __call__
  response = self.app(environ, start_response)
File '/srv/ckan/src/ckan/ckan/config/middleware/common_middleware.py', line 28 in __call__
  return self.app(environ, start_response)
File '/usr/lib/python2.7/site-packages/pylons/wsgiapp.py', line 125 in __call__
  response = self.dispatch(controller, environ, start_response)
File '/usr/lib/python2.7/site-packages/pylons/wsgiapp.py', line 324 in dispatch
  return controller(environ, start_response)
File '/srv/ckan/src/ckan/ckan/lib/base.py', line 231 in __call__
  res = WSGIController.__call__(self, environ, start_response)
File '/usr/lib/python2.7/site-packages/pylons/controllers/core.py', line 221 in __call__
  response = self._dispatch_call()
File '/usr/lib/python2.7/site-packages/pylons/controllers/core.py', line 172 in _dispatch_call
  response = self._inspect_call(func)
File '/usr/lib/python2.7/site-packages/pylons/controllers/core.py', line 107 in _inspect_call
  result = self._perform_call(func, args)
File '/usr/lib/python2.7/site-packages/pylons/controllers/core.py', line 60 in _perform_call
  return func(**args)
File '/srv/ckan/src/ckan/ckanext/datapusher/plugin.py', line 71 in resource_data
  extra_vars={'status': datapusher_status})
File '/srv/ckan/src/ckan/ckan/lib/base.py', line 195 in render
  return cached_template(template_name, render_template)
File '/usr/lib/python2.7/site-packages/pylons/templating.py', line 249 in cached_template
  return render_func()
File '/srv/ckan/src/ckan/ckan/lib/base.py', line 137 in render_template
  return render_jinja2(template_name, globs)
File '/srv/ckan/src/ckan/ckan/lib/base.py', line 94 in render_jinja2
  return template.render(**extra_vars)
File '/usr/lib/python2.7/site-packages/jinja2/environment.py', line 989 in render
  return self.environment.handle_exception(exc_info, True)
File '/usr/lib/python2.7/site-packages/jinja2/environment.py', line 754 in handle_exception
  reraise(exc_type, exc_value, tb)
File '/srv/ckan/src/ckan/ckanext/datapusher/templates/datapusher/resource_data.html', line 1 in top-level template code
  {% extends "package/resource_edit_base.html" %}
File '/srv/ckan/src/ckan/ckanext/datastore/templates/package/resource_edit_base.html', line 1 in top-level template code
  {% ckan_extends %}
File '/srv/ckan/src/ckan/ckanext/datapusher/templates/package/resource_edit_base.html', line 1 in top-level template code
  {% ckan_extends %}
File '/srv/ckan/src/ckan/ckan/templates/package/resource_edit_base.html', line 4 in top-level template code
  {% set res = c.resource %}
File '/srv/ckan/src/ckan/ckan/templates/package/base.html', line 3 in top-level template code
  {% set pkg = c.pkg_dict or pkg_dict %}
File '/srv/ckan/src/ckan/ckan/templates/page.html', line 1 in top-level template code
  {% extends "base.html" %}
File '/srv/ckan/src/ckanext-geoview/ckanext/geoview/templates/base.html', line 1 in top-level template code
  {% ckan_extends %}
File '/srv/ckan/src/ckanext-pages/ckanext/pages/theme/templates_main/base.html', line 1 in top-level template code
  {% ckan_extends %}
File '/srv/ckan/src/ckan/ckan/templates/base.html', line 101 in top-level template code
  {%- block page %}{% endblock -%}
File '/srv/ckan/src/ckan/ckan/templates/page.html', line 19 in block "page"
  {%- block content %}
File '/srv/ckan/src/ckan/ckan/templates/page.html', line 22 in block "content"
  {% block main_content %}
File '/srv/ckan/src/ckan/ckan/templates/page.html', line 74 in block "main_content"
  {% block primary %}
File '/srv/ckan/src/ckan/ckan/templates/page.html', line 87 in block "primary"
  {% block primary_content %}
File '/srv/ckan/src/ckan/ckan/templates/page.html', line 107 in block "primary_content"
  {% block primary_content_inner %}
File '/srv/ckan/src/ckan/ckanext/datapusher/templates/datapusher/resource_data.html', line 76 in block "primary_content_inner"
  <a href="#" data-target="popover" data-content="<dl>{% for key, value in item.iteritems() %}<dt>{{ key }}</dt><dd>{{ h.clean_html(value) }}</dd>{% endfor %}</dl>" data-html="true">{{ _('Details') }}</a>
File '/usr/lib/python2.7/site-packages/bleach/__init__.py', line 99 in clean
  return cleaner.clean(text)
File '/usr/lib/python2.7/site-packages/bleach/sanitizer.py', line 203 in clean
  raise TypeError(message)
TypeError: argument cannot be of 'datetime' type, must be of text type

```

### What steps can be taken to reproduce the issue? 
1. Install CKAN with PR #3952 merged (any commit after 4892cf39bc889fd100b4a4e44f5c452513c76339).
2. Install DataPusher and configure CKAN to work with it.
3. Upload a resource.


Discussion:
@davidread #3952 was your PR would you mind having a look?

Resolving patch:
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -25,7 +25,7 @@
 import webhelpers.text as whtext
 import webhelpers.date as date
 from markdown import markdown
-from bleach import clean as clean_html, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
+from bleach import clean as bleach_clean, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
 from pylons import url as _pylons_default_url
 from ckan.common import config, is_flask_request
 from flask import redirect as _flask_redirect
@@ -2102,7 +2102,7 @@ def render_markdown(data, auto_link=True, allow_html=False):
         data = markdown(data.strip())
     else:
         data = RE_MD_HTML_TAGS.sub('', data.strip())
-        data = clean_html(
+        data = bleach_clean(
             markdown(data), strip=True,
             tags=MARKDOWN_TAGS, attributes=MARKDOWN_ATTRIBUTES)
     # tags can be added by tag:... or tag:"...." and a link will be made
@@ -2555,6 +2555,11 @@ def radio(selected, id, checked):
         value="%s" type="radio">') % (selected, id, selected, id))
 
 
+@core_helper
+def clean_html(html):
+    return bleach_clean(unicode(html))
+
+
 core_helper(flash, name='flash')
 core_helper(localised_number)
 core_helper(localised_SI_number)
@@ -2573,7 +2578,6 @@ def radio(selected, id, checked):
 core_helper(converters.asbool)
 # Useful additions from the stdlib.
 core_helper(urlencode)
-core_helper(clean_html, name='clean_html')
 
 
 def load_plugin_helpers():


Test patch:
diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py
--- a/ckan/tests/lib/test_helpers.py
+++ b/ckan/tests/lib/test_helpers.py
@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+import datetime
+
 import nose
 import pytz
 import tzlocal
@@ -525,6 +527,17 @@ def test_named_timezone(self):
         eq_(h.get_display_timezone(), pytz.timezone('America/New_York'))
 
 
+class TestCleanHtml(object):
+    def test_disallowed_tag(self):
+        eq_(h.clean_html('<b><bad-tag>Hello'),
+            u'<b>&lt;bad-tag&gt;Hello&lt;/bad-tag&gt;</b>')
+
+    def test_non_string(self):
+        # allow a datetime for compatibility with older ckanext-datapusher
+        eq_(h.clean_html(datetime.datetime(2018, 1, 5, 10, 48, 23, 463511)),
+            u'2018-01-05 10:48:23.463511')
+
+
 class TestHelperException(helpers.FunctionalTestBase):
 
     @raises(ckan.exceptions.HelperError)