Repository: OCHA-DAP/hdx-ckan

Issue:
Adding fields pushed from CPS to dataset schema
Also:
- change list of fields allowed to be modified by `hdx_package_update_metadata` to include the fields pushed by CPS
- change tests to check that the fields pushed by CPS are modifiable



Discussion:

Resolving patch:
diff --git a/ckanext-hdx_package/ckanext/hdx_package/helpers/update.py b/ckanext-hdx_package/ckanext/hdx_package/helpers/update.py
--- a/ckanext-hdx_package/ckanext/hdx_package/helpers/update.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/helpers/update.py
@@ -229,11 +229,15 @@ def hdx_package_update_metadata(context, data_dict):
 
     allowed_fields = ['indicator', 'package_creator', 'methodology',
                       'dataset_source', 'dataset_date', 'license_other',
-                      'license_title', 'caveats', 'name', 'title']
+                      'license_title', 'caveats', 'name', 'title',
+                      'last_metadata_update_date', 'dataset_source_code', 'dataset_source',
+                      'indicator_type', 'indicator_type_code', 'dataset_summary',
+                      'methodology', 'more_info', 'terms_of_use',
+                      'validation_notes_and_comments', 'last_data_update_date']
 
     package = _get_action('package_show')(context, data_dict)
     for key, value in data_dict.iteritems():
-        if key != 'id':
+        if key in allowed_fields:
             package[key] = value
     if not package['notes']:
         package['notes'] = ' '
@@ -251,7 +255,8 @@ def hdx_resource_update_metadata(context, data_dict):
 
     resource = _get_action('resource_show')(context, data_dict)
     for key, value in data_dict.iteritems():
-        resource[key] = value
+        if key in allowed_fields:
+            resource[key] = value
     resource = _get_action('resource_update')(context, resource)
 
     return resource
diff --git a/ckanext-hdx_package/ckanext/hdx_package/plugin.py b/ckanext-hdx_package/ckanext/hdx_package/plugin.py
--- a/ckanext-hdx_package/ckanext/hdx_package/plugin.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/plugin.py
@@ -109,8 +109,26 @@ def _modify_package_schema(self, schema):
             'groups_list': [vd.groups_not_empty],
             'indicator': [tk.get_validator('ignore_missing'),
                           tk.get_converter('convert_to_extras')],
+            'last_data_update_date': [tk.get_validator('ignore_missing'),
+                                      tk.get_converter('convert_to_extras')],
+            'last_metadata_update_date': [tk.get_validator('ignore_missing'),
+                                          tk.get_converter('convert_to_extras')],
+            'dataset_source_code': [tk.get_validator('ignore_missing'),
+                                    tk.get_converter('convert_to_extras')],
+            'indicator_type': [tk.get_validator('ignore_missing'),
+                               tk.get_converter('convert_to_extras')],
+            'indicator_type_code': [tk.get_validator('ignore_missing'),
+                                    tk.get_converter('convert_to_extras')],
             'dataset_summary': [tk.get_validator('ignore_missing'),
                                 tk.get_converter('convert_to_extras')],
+            'methodology': [tk.get_validator('ignore_missing'),
+                            tk.get_converter('convert_to_extras')],
+            'more_info': [tk.get_validator('ignore_missing'),
+                          tk.get_converter('convert_to_extras')],
+            'terms_of_use': [tk.get_validator('ignore_missing'),
+                             tk.get_converter('convert_to_extras')],
+            'validation_notes_and_comments': [tk.get_validator('ignore_missing'),
+                                              tk.get_converter('convert_to_extras')],
             'caveats': [tk.get_validator('ignore_missing'),
                         tk.get_converter('convert_to_extras')],
             'dataset_source': [tk.get_validator('not_empty'),
@@ -146,8 +164,26 @@ def show_package_schema(self):
                                 tk.get_validator('ignore_missing')],
             'indicator': [tk.get_converter('convert_from_extras'),
                           tk.get_validator('ignore_missing')],
+            'last_data_update_date': [tk.get_converter('convert_from_extras'),
+                                      tk.get_validator('ignore_missing')],
+            'last_metadata_update_date': [tk.get_converter('convert_from_extras'),
+                                          tk.get_validator('ignore_missing')],
+            'dataset_source_code': [tk.get_converter('convert_from_extras'),
+                                    tk.get_validator('ignore_missing')],
+            'indicator_type': [tk.get_converter('convert_from_extras'),
+                               tk.get_validator('ignore_missing')],
+            'indicator_type_code': [tk.get_converter('convert_from_extras'),
+                                    tk.get_validator('ignore_missing')],
             'dataset_summary': [tk.get_converter('convert_from_extras'),
                                 tk.get_validator('ignore_missing')],
+            'methodology': [tk.get_converter('convert_from_extras'),
+                            tk.get_validator('ignore_missing')],
+            'more_info': [tk.get_converter('convert_from_extras'),
+                          tk.get_validator('ignore_missing')],
+            'terms_of_use': [tk.get_converter('convert_from_extras'),
+                             tk.get_validator('ignore_missing')],
+            'validation_notes_and_comments': [tk.get_converter('convert_from_extras'),
+                                              tk.get_validator('ignore_missing')],
             'caveats': [tk.get_converter('convert_from_extras'),
                         tk.get_validator('ignore_missing')],
             'dataset_source': [tk.get_converter('convert_from_extras'),


Test patch:
diff --git a/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_package_update.py b/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_package_update.py
--- a/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_package_update.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_package_update.py
@@ -58,7 +58,20 @@ def test_hdx_package_update_metadata(self):
         modified_fields = {'id': 'test_activity_1', 'name': 'test_activity_1_modified',
                            'indicator': '2',
                            'title': "Modified Test Activity 1",
-                           'dataset_source': 'Modified source'}
+                           'dataset_source': 'Modified source',
+                           'last_metadata_update_date': 'last_metadata_update_date test',
+                           'last_data_update_date': 'last_data_update_date test',
+                           'dataset_date': '11/02/2014-11/20/2014',
+                           'dataset_source': 'dataset_source test',
+                           'dataset_source_code': 'dataset_source_code test',
+                           'indicator_type': 'indicator_type test',
+                           'indicator_type_code': 'indicator_type_code test',
+                           'dataset_summary': 'dataset_summary test',
+                           'methodology': 'methodology test',
+                           'more_info': 'more_info test',
+                           'terms_of_use': 'terms_of_use test',
+                           'validation_notes_and_comments': 'validation_notes_and_comments test'
+                           }
 
         self._get_action('hdx_package_update_metadata')(
             context, modified_fields)
@@ -70,13 +83,21 @@ def test_hdx_package_update_metadata(self):
 
         modified_fields.pop('id')
 
-        for key, value in package.iteritems():
+        # Checking that all fields in the modified_package come either
+        # from original package or were modified
+        for key, value in modified_package.iteritems():
             if key not in modified_fields.keys():
-                if key != 'groups':
-                    assert modified_package[key] == value, 'Problem with key {}: has value {} instead of {}'.format(
-                        key, modified_package[key], value)
+                if key != 'groups' and key in package:
+                    assert package[key] == value, 'Problem with key {}: has value {} instead of {}'.format(
+                        key, value, package[key])
             else:
-                assert modified_package[key] == modified_fields[key], 'Problem with key {}: has value {} instead of {}'.format(
-                    key, modified_package[key], modified_fields[key])
+                assert value == modified_fields[key], 'Problem with key {}: has value {} instead of {}'.format(
+                    key, value, modified_fields[key])
+
+        # Checking that all modifications were applied
+        for key, value in modified_fields.iteritems():
+            assert value == modified_package[key], 'Problem with key {}: has value {} instead of {}'.format(
+                key, modified_package[key], value)
+
         assert len(modified_package['groups']) == len(package['groups']), 'There should be {} item in groups but instead there is {}'.format(
             len(package['groups']), len(modified_package['groups']))