Repository: ckan/ckan

Issue:
KeyError: 'key' after Patch Update to 2.8.5
**CKAN version**
2.8.5
**Describe the bug**
After Patch Update to 2.8.5 my CKAN catalog is showing an internal server error when I´m trying to open a dataset list or dataset. 
The ckan_default.error.log is showing KeyErrors

**Steps to reproduce**
Trying to do the latest patch update from CKAN version 2.8.4 to 2.8.5 on a ubuntu 18.04 instance. Because my CKAN installation is from source I tried following steps:
`. /usr/lib/ckan/default/bin/activate`
`cd /usr/lib/ckan/default/src/ckan`
`git fetch` 
`git checkout ckan-2.8.5`
`pip install --upgrade -r requirements.txt`
`paster search-index rebuild -r --config=/etc/ckan/default/production.ini`
`sudo service apache2 restart `


**Expected behavior**
I should be able to open any dataset-list or dataset via browser in the CKAN catalog

**Additional details**
full Error from ckan_default.error.log:
```
Error - <type 'exceptions.KeyError'>: 'key'
 URL: http://apgc.awi.de/group/1fad2eb2-9ee5-4030-80f1-4563a803525c?organization=pangaea&res_format=FileGDB
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/weberror/errormiddleware.py', line 171 in __call__
   app_iter = self.application(environ, sr_checker)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/dec.py', line 147 in __call__
   resp = self.call_func(req, *args, **self.kwargs)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/dec.py', line 208 in call_func
   return self.func(req, *args, **kwargs)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/fanstatic/publisher.py', line 234 in __call__
   return request.get_response(self.app)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/request.py', line 1053 in get_response
   application, catch_exc_info=False)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/request.py', line 1022 in call_application
   app_iter = application(self.environ, start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/dec.py', line 147 in __call__
   resp = self.call_func(req, *args, **self.kwargs)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/dec.py', line 208 in call_func
   return self.func(req, *args, **kwargs)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/fanstatic/injector.py', line 54 in __call__
   response = request.get_response(self.app)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/request.py', line 1053 in get_response
   application, catch_exc_info=False)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/webob/request.py', line 1022 in call_application
   app_iter = application(self.environ, start_response)
 File '/usr/lib/ckan/default/src/ckan/ckan/config/middleware/pylons_app.py', line 264 in inner
   result = application(environ, start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/beaker/middleware.py', line 73 in __call__
   return self.app(environ, start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/beaker/middleware.py', line 156 in __call__
   return self.wrap_app(environ, session_start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/routes/middleware.py', line 131 in __call__
   response = self.app(environ, start_response)
 File '/usr/lib/ckan/default/src/ckan/ckan/config/middleware/common_middleware.py', line 33 in __call__
   return self.app(environ, start_response)
 File '/usr/lib/ckan/default/src/ckan/ckan/config/middleware/common_middleware.py', line 59 in __call__
   return self.app(environ, start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/wsgiapp.py', line 125 in __call__
   response = self.dispatch(controller, environ, start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/wsgiapp.py', line 324 in dispatch
   return controller(environ, start_response)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 242 in __call__
   res = WSGIController.__call__(self, environ, start_response)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/controllers/core.py', line 221 in __call__
   response = self._dispatch_call()
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/controllers/core.py', line 172 in _dispatch_call
   response = self._inspect_call(func)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/controllers/core.py', line 107 in _inspect_call
   result = self._perform_call(func, args)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/controllers/core.py', line 60 in _perform_call
   return func(**args)
 File '/usr/lib/ckan/default/src/ckan/ckan/controllers/group.py', line 230 in read
   extra_vars={'group_type': group_type})
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 127 in render
   return cached_template(template_name, renderer)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/templating.py', line 249 in cached_template
   return render_func()
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 164 in render_template
   return render_jinja2(template_name, globs)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 96 in render_jinja2
   return template.render(**extra_vars)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/jinja2/environment.py', line 989 in render
   return self.environment.handle_exception(exc_info, True)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/jinja2/environment.py', line 754 in handle_exception
   reraise(exc_type, exc_value, tb)
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/group/read.html', line 1 in top-level template code
   {% extends "group/read_base.html" %}
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/group/read_base.html', line 1 in top-level template code
   {% ckan_extends %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/group/read_base.html', line 1 in top-level template code
   {% extends "page.html" %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/page.html', line 1 in top-level template code
   {% extends "base.html" %}
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/ckanext/geoview/templates/base.html', line 1 in top-level template code
   {% ckan_extends %}
 File '/usr/lib/ckan/default/src/ckanext-pages/ckanext/pages/theme/templates_main/base.html', line 1 in top-level template code
   {% ckan_extends %}
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/base.html', line 1 in top-level template code
   {% ckan_extends %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/base.html', line 101 in top-level template code
   {%- block page %}{% endblock -%}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/page.html', line 19 in block "page"
   {%- block content %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/page.html', line 22 in block "content"
   {% block main_content %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/page.html', line 74 in block "main_content"
   {% block primary %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/page.html', line 87 in block "primary"
   {% block primary_content %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/page.html', line 107 in block "primary_content"
   {% block primary_content_inner %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/group/read.html', line 21 in block "primary_content_inner"
   {% block packages_list %}
 File '/usr/lib/ckan/default/src/ckan/ckan/templates/group/read.html', line 23 in block "packages_list"
   {{ h.snippet('snippets/package_list.html', packages=c.page.items) }}
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/helpers.py', line 1709 in snippet
   return base.render_snippet(template_name, **kw)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 86 in render_snippet
   output = render(template_name, extra_vars=kw)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 127 in render
   return cached_template(template_name, renderer)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/templating.py', line 249 in cached_template
   return render_func()
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 164 in render_template
   return render_jinja2(template_name, globs)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 96 in render_jinja2
   return template.render(**extra_vars)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/jinja2/environment.py', line 989 in render
   return self.environment.handle_exception(exc_info, True)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/jinja2/environment.py', line 754 in handle_exception
   reraise(exc_type, exc_value, tb)
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/snippets/package_list.html', line 17 in top-level template code
   {% block package_list %}
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/snippets/package_list.html', line 20 in block "package_list"
   {% block package_list_inner %}
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/snippets/package_list.html', line 22 in block "package_list_inner"
   {% snippet 'snippets/package_item.html', package=package, item_class=item_class, hide_resources=hide_resources, banner=banner, truncate=truncate , truncate_title=120%}
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/jinja_extensions.py', line 268 in _call
   return base.render_snippet(args[0], **kwargs)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 86 in render_snippet
   output = render(template_name, extra_vars=kw)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 127 in render
   return cached_template(template_name, renderer)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/pylons/templating.py', line 249 in cached_template
   return render_func()
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 164 in render_template
   return render_jinja2(template_name, globs)
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/base.py', line 96 in render_jinja2
   return template.render(**extra_vars)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/jinja2/environment.py', line 989 in render
   return self.environment.handle_exception(exc_info, True)
 File '/usr/lib/ckan/default/lib/python2.7/site-packages/jinja2/environment.py', line 754 in handle_exception
   reraise(exc_type, exc_value, tb)
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/snippets/package_item.html', line 39 in top-level template code
   {% block package_item %}
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/snippets/package_item.html', line 41 in block "package_item"
   {% block content %}
 File '/usr/lib/ckan/default/src/ckanext-apgc_theme/ckanext/apgc_theme/templates/snippets/package_item.html', line 81 in block "content"
   {% for key, value in h.sorted_extras(package.extras) %}
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/helpers.py', line 1135 in sorted_extras
   for extra in sorted(package_extras, key=lambda x: x['key']):
 File '/usr/lib/ckan/default/src/ckan/ckan/lib/helpers.py', line 1135 in <lambda>
   for extra in sorted(package_extras, key=lambda x: x['key']):
 KeyError: 'key'
```



Discussion:

Resolving patch:
diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py
--- a/ckan/logic/converters.py
+++ b/ckan/logic/converters.py
@@ -26,14 +26,24 @@ def convert_to_extras(key, data, errors, context):
 
 def convert_from_extras(key, data, errors, context):
 
-    def remove_from_extras(data, key):
-        to_remove = []
-        for data_key, data_value in six.iteritems(data):
-            if (data_key[0] == 'extras'
-                and data_key[1] == key):
-                to_remove.append(data_key)
-        for item in to_remove:
-            del data[item]
+    def remove_from_extras(data, idx):
+        for key in sorted(data):
+            if key[0] != 'extras' or key[1] < idx:
+                continue
+            if key[1] == idx:
+                del data[key]
+
+            # Following block required for unflattening extras with
+            # "gaps" created sometimes by `convert_from_extra`
+            # validator :
+            #
+            #   {
+            #     ('extras', 0, 'key'): 'x',
+            #     ('extras', 2, 'key): 'y'
+            #   }
+            if key[1] > idx:
+                new_key = (key[0], key[1] - 1) + key[2:]
+                data[new_key] = data.pop(key)
 
     for data_key, data_value in six.iteritems(data):
         if (data_key[0] == 'extras'
diff --git a/ckanext/example_idatasetform/plugin_v5.py b/ckanext/example_idatasetform/plugin_v5.py
--- a/ckanext/example_idatasetform/plugin_v5.py
+++ b/ckanext/example_idatasetform/plugin_v5.py
@@ -30,7 +30,9 @@ def show_package_schema(self):
         schema = super(ExampleIDatasetFormPlugin, self).show_package_schema()
         schema.update({
             u'custom_text': [tk.get_converter(u'convert_from_extras'),
-                             tk.get_validator(u'ignore_missing')]
+                             tk.get_validator(u'ignore_missing')],
+            u'custom_text_2': [tk.get_converter(u'convert_from_extras'),
+                               tk.get_validator(u'ignore_missing')],
         })
         return schema
 


Test patch:
diff --git a/ckanext/example_idatasetform/tests/test_example_idatasetform.py b/ckanext/example_idatasetform/tests/test_example_idatasetform.py
--- a/ckanext/example_idatasetform/tests/test_example_idatasetform.py
+++ b/ckanext/example_idatasetform/tests/test_example_idatasetform.py
@@ -79,6 +79,40 @@ def test_custom_package_type_urls(self):
             url_for("fancy_type.edit", id="check") == "/fancy_type/edit/check"
         )
 
+    def test_custom_field_with_extras(self):
+        dataset = factories.Dataset(
+            type='fancy_type',
+            name='test-dataset',
+            custom_text='custom-text',
+            extras=[
+                {'key': 'key1', 'value': 'value1'},
+                {'key': 'key2', 'value': 'value2'},
+            ]
+        )
+        assert dataset['custom_text'] == 'custom-text'
+        assert dataset['extras'] == [
+            {'key': 'key1', 'value': 'value1'},
+            {'key': 'key2', 'value': 'value2'},
+        ]
+
+    def test_mixed_extras(self):
+        dataset = factories.Dataset(
+            type='fancy_type',
+            name='test-dataset',
+            custom_text='custom-text',
+            extras=[
+                {'key': 'key1', 'value': 'value1'},
+                {'key': 'custom_text_2', 'value': 'custom-text-2'},
+                {'key': 'key2', 'value': 'value2'},
+            ],
+        )
+        assert dataset['custom_text'] == 'custom-text'
+        assert dataset['custom_text_2'] == 'custom-text-2'
+        assert dataset['extras'] == [
+            {'key': 'key1', 'value': 'value1'},
+            {'key': 'key2', 'value': 'value2'},
+        ]
+
 
 @pytest.mark.ckan_config("ckan.plugins", u"example_idatasetform_v5")
 @pytest.mark.ckan_config("package_edit_return_url", None)