Repository: datosgobar/pydatajson

Issue:
Robustecer el manejo de themes por parte de push_dataset_to_ckan()

Este es un issue que hace referencia al 24 de monitoreo apertura:
https://github.com/datosgobar/monitoreo-apertura/issues/24
Lo abro acá porque me parece que es más apropiado discutirlo en este repo.

--------------------------------------------------------------------------------------------------
**Contexto**

Se intentó federar el dataset de series de tiempo a la instancia andino.datos.gob.ar, de la siguiente manera:
```
catalog_modernizacion = DataJson("http://infra.datos.gob.ar/catalog/modernizacion/data.json")

catalog_modernizacion.push_dataset_to_ckan(
    "modernizacion", "ministerio-de-modernizacion", "7",
    portal_url, apikey
)
```
y dio la rta:
```
---------------------------------------------------------------------------
StopIteration                             Traceback (most recent call last)
<ipython-input-8-b4c2dae3c9ec> in <module>()
      1 catalog_modernizacion.push_dataset_to_ckan(
      2     "modernizacion", "ministerio-de-modernizacion", "7",
----> 3     portal_url, apikey
      4 )

/Users/abenassi/github/pydatajson/pydatajson/federation.pyc in push_dataset_to_ckan(catalog, catalog_id, owner_org, dataset_origin_identifier, portal_url, apikey)
     49         theme_taxonomy = catalog.themes
     50         for theme in themes:
---> 51             label = next(x['label'] for x in theme_taxonomy if x['id'] == theme)
     52             package['tags'].append({'name': label})
     53 

StopIteration: 
```

**Propuesta**

Investigar qué hizo que se frenara la operación. Puede ser que el problema sea que se requiere crear "temas" que la instancia destino no tiene. Si es así, lo mejor sería incorporar la creación automática de temas que no existan dentro de la operación de federación.

Discussion:

Patch:
diff --git a/pydatajson/ckan_utils.py b/pydatajson/ckan_utils.py
index f71570d..9724f44 100644
--- a/pydatajson/ckan_utils.py
+++ b/pydatajson/ckan_utils.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import json
+import re
+import sys
 from datetime import time
 from dateutil import parser, tz
 from .helpers import title_to_name
@@ -14,7 +16,7 @@ def append_attribute_to_extra(package, dataset, attribute, serialize=False):
         package['extras'].append({'key': attribute, 'value': value})
 
 
-def map_dataset_to_package(dataset, owner_org, theme_taxonomy, catalog_id=None,
+def map_dataset_to_package(catalog, dataset, owner_org, catalog_id=None,
                            demote_superThemes=True, demote_themes=True):
     package = dict()
     package['extras'] = []
@@ -66,7 +68,8 @@ def map_dataset_to_package(dataset, owner_org, theme_taxonomy, catalog_id=None,
     if themes and demote_themes:
         package['tags'] = package.get('tags', [])
         for theme in themes:
-            label = next(x['label'] for x in theme_taxonomy if x['id'] == theme)
+            label = catalog.get_theme(identifier=theme)['label']
+            label = re.sub(r'[^\wá-úÁ-ÚñÑ .-]+', '', label, flags=re.UNICODE)
             package['tags'].append({'name': label})
     else:
         package['groups'] = package.get('groups', []) + [{'name': title_to_name(theme, decode=False)}
diff --git a/pydatajson/federation.py b/pydatajson/federation.py
index 2c31623..9573040 100644
--- a/pydatajson/federation.py
+++ b/pydatajson/federation.py
@@ -29,9 +29,8 @@ def push_dataset_to_ckan(catalog, owner_org, dataset_origin_identifier, portal_u
     """
     dataset = catalog.get_dataset(dataset_origin_identifier)
     ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
-    theme_taxonomy = catalog.themes
 
-    package = map_dataset_to_package(dataset, owner_org, theme_taxonomy, catalog_id,
+    package = map_dataset_to_package(catalog, dataset, owner_org, catalog_id,
                                      demote_superThemes, demote_themes)
 
     # Get license id
diff --git a/pydatajson/search.py b/pydatajson/search.py
index 0ab6391..1c3d25a 100644
--- a/pydatajson/search.py
+++ b/pydatajson/search.py
@@ -280,7 +280,7 @@ def get_theme(catalog, identifier=None, label=None):
 
     # filtra por id (preferentemente) o label
     if identifier:
-        filtered_themes = [theme for theme in themes if theme["id"] == identifier]
+        filtered_themes = [theme for theme in themes if theme["id"].lower() == identifier.lower()]
         if len(filtered_themes) > 1:
             raise ThemeIdRepeated([x["id"] for x in filtered_themes])
 


Test patch:
diff --git a/tests/samples/full_data.json b/tests/samples/full_data.json
index b349578..e80673c 100644
--- a/tests/samples/full_data.json
+++ b/tests/samples/full_data.json
@@ -193,7 +193,7 @@
             "id": "convocatorias"
         },
         {
-            "label": "Compras",
+            "label": "Adquisición",
             "description": "Datasets sobre compras realizadas.",
             "id": "compras"
         },
@@ -213,7 +213,7 @@
             "id": "normativa"
         },
         {
-            "label": "Proveedores",
+            "label": "Proveeduría",
             "description": "Datasets sobre proveedores del Estado.",
             "id": "proveedores"
         }
diff --git a/tests/test_ckan_utils.py b/tests/test_ckan_utils.py
index 3f20984..f90406e 100644
--- a/tests/test_ckan_utils.py
+++ b/tests/test_ckan_utils.py
@@ -1,6 +1,10 @@
+# -*- coding: utf-8 -*-
+
 import unittest
 import os
 import json
+import re
+import sys
 from dateutil import parser, tz
 from .context import pydatajson
 from pydatajson.ckan_utils import map_dataset_to_package, map_distributions_to_resources, convert_iso_string_to_utc
@@ -23,15 +27,15 @@ class DatasetConversionTestCase(unittest.TestCase):
         cls.distributions = cls.dataset['distribution']
 
     def test_catalog_id_is_prepended_to_dataset_id_if_passed(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes, catalog_id=self.catalog_id)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner', catalog_id=self.catalog_id)
         self.assertEqual(self.catalog_id + '_' + self.dataset_id, package['id'])
 
     def test_dataset_id_is_preserved_if_catlog_id_is_not_passed(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner')
         self.assertEqual(self.dataset_id, package['id'])
 
     def test_replicated_plain_attributes_are_corrext(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes, catalog_id=self.catalog_id)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner', catalog_id=self.catalog_id)
         plain_replicated_attributes = [('title', 'title'),
                                        ('notes', 'description'),
                                        ('url', 'landingPage')]
@@ -40,7 +44,7 @@ class DatasetConversionTestCase(unittest.TestCase):
         self.assertEqual('owner', package['owner_org'])
 
     def test_dataset_nested_replicated_attributes_stay_the_same(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes, catalog_id=self.catalog_id)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner', catalog_id=self.catalog_id)
         contact_point_nested = [('maintainer', 'fn'),
                                 ('maintainer_email', 'hasEmail')]
         for fst, snd in contact_point_nested:
@@ -51,7 +55,7 @@ class DatasetConversionTestCase(unittest.TestCase):
             self.assertEqual(self.dataset.get('publisher').get(snd), package.get(fst))
 
     def test_dataset_array_attributes_are_correct(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes, catalog_id=self.catalog_id)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner', catalog_id=self.catalog_id)
         groups = [group['name'] for group in package.get('groups', [])]
         super_themes = [title_to_name(s_theme.lower()) for s_theme in self.dataset.get('superTheme')]
         try:
@@ -65,7 +69,8 @@ class DatasetConversionTestCase(unittest.TestCase):
         themes = self.dataset.get('theme', [])
         theme_labels = []
         for theme in themes:
-            label = next(x['label'] for x in self.catalog.themes if x['id'] == theme)
+            label = self.catalog.get_theme(identifier=theme)['label']
+            label = re.sub(r'[^\w .-]+', '', label, flags=re.UNICODE)
             theme_labels.append(label)
 
         try:
@@ -74,7 +79,7 @@ class DatasetConversionTestCase(unittest.TestCase):
             self.assertCountEqual(keywords + theme_labels, tags)
 
     def test_themes_are_preserved_if_not_demoted(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes,
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner',
                                          catalog_id=self.catalog_id, demote_themes=False)
         groups = [group['name'] for group in package.get('groups', [])]
         super_themes = [title_to_name(s_theme.lower()) for s_theme in self.dataset.get('superTheme')]
@@ -92,7 +97,7 @@ class DatasetConversionTestCase(unittest.TestCase):
             self.assertCountEqual(keywords, tags)
 
     def test_superThemes_dont_impact_groups_if_not_demoted(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes,
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner',
                                          catalog_id=self.catalog_id, demote_superThemes=False)
         groups = [group['name'] for group in package.get('groups', [])]
         tags = [tag['name'] for tag in package['tags']]
@@ -100,7 +105,8 @@ class DatasetConversionTestCase(unittest.TestCase):
         themes = self.dataset.get('theme', [])
         theme_labels = []
         for theme in themes:
-            label = next(x['label'] for x in self.catalog.themes if x['id'] == theme)
+            label = self.catalog.get_theme(identifier=theme)['label']
+            label = re.sub(r'[^\wá-úÁ-ÚñÑ .-]+', '', label, flags=re.UNICODE)
             theme_labels.append(label)
         try:
             self.assertItemsEqual([], groups)
@@ -112,7 +118,7 @@ class DatasetConversionTestCase(unittest.TestCase):
             self.assertCountEqual(keywords + theme_labels, tags)
 
     def test_preserve_themes_and_superThemes(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes,
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner',
                                          self.catalog_id, False, False)
         groups = [group['name'] for group in package.get('groups', [])]
         tags = [tag['name'] for tag in package['tags']]
@@ -128,7 +134,7 @@ class DatasetConversionTestCase(unittest.TestCase):
             self.assertCountEqual(keywords, tags)
 
     def test_dataset_extra_attributes_are_correct(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes, catalog_id=self.catalog_id)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner', catalog_id=self.catalog_id)
 #       extras are included in dataset
         if package['extras']:
             for extra in package['extras']:
@@ -144,7 +150,7 @@ class DatasetConversionTestCase(unittest.TestCase):
                     self.assertEqual(dataset_value, extra_value)
 
     def test_dataset_extra_attributes_are_complete(self):
-        package = map_dataset_to_package(self.dataset, 'owner', self.catalog.themes, catalog_id=self.catalog_id)
+        package = map_dataset_to_package(self.catalog, self.dataset, 'owner', catalog_id=self.catalog_id)
 #       dataset attributes are included in extras
         extra_attrs = ['issued', 'modified', 'accrualPeriodicity', 'temporal', 'language', 'spatial', 'superTheme']
         for key in extra_attrs:
diff --git a/tests/test_federation.py b/tests/test_federation.py
index e4a1d2e..e6804b9 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -1,6 +1,9 @@
+# -*- coding: utf-8 -*-
+
 import unittest
 import os
 import re
+import sys
 try:
     from mock import patch, MagicMock
 except ImportError:
@@ -83,10 +86,10 @@ class PushDatasetTestCase(unittest.TestCase):
     @patch('pydatajson.federation.RemoteCKAN', autospec=True)
     def test_tags_are_passed_correctly(self, mock_portal):
         themes = self.dataset['theme']
-        theme_taxonomy = self.catalog.themes
         keywords = [kw for kw in self.dataset['keyword']]
         for theme in themes:
-            label = next(x['label'] for x in theme_taxonomy if x['id'] == theme)
+            label = self.catalog.get_theme(identifier=theme)['label']
+            label = re.sub(r'[^\w .-]+', '', label, flags=re.UNICODE)
             keywords.append(label)
 
         def mock_call_action(action, data_dict=None):