Repository: datosgobar/pydatajson

Issue:
Robustecer el manejo de harvest_catalog_to_ckan()
Hay que corregir 2 problemas:

- En caso de pasar una lista vacía en el dataset list, no se debe federar ningún dataset. Actualmente se federan todos.

-En caso de que alguno de las llamadas a `harvest_dataset_to_ckan()` falle, loggear y continuar con el resto. Actualmente la federación entera del catálogo levanta la excepción.

Comments:

Patch:
diff --git a/pydatajson/ckan_utils.py b/pydatajson/ckan_utils.py
index 6df8eaa..c67d3dc 100644
--- a/pydatajson/ckan_utils.py
+++ b/pydatajson/ckan_utils.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from __future__ import print_function
+
 import json
 import re
 from datetime import time
@@ -76,12 +78,13 @@ def map_dataset_to_package(catalog, dataset, owner_org, catalog_id=None,
     if themes and demote_themes:
         package['tags'] = package.get('tags', [])
         for theme in themes:
+            # si falla continúa sin agregar ese theme a los tags del dataset
             try:
-                label = catalog.get_theme(identifier=theme)['label']
-            except:
-                label = catalog.get_theme(label=theme)['label']
-            label = re.sub(r'[^\wá-úÁ-ÚñÑ .-]+', '', label, flags=re.UNICODE)
-            package['tags'].append({'name': label})
+                label = _get_theme_label(catalog, theme)
+                package['tags'].append({'name': label})
+            except Exception as e:
+                print(e)
+                continue
     else:
         package['groups'] = package.get('groups', []) + [
             {'name': title_to_name(theme, decode=False)}
@@ -91,6 +94,18 @@ def map_dataset_to_package(catalog, dataset, owner_org, catalog_id=None,
     return package
 
 
+def _get_theme_label(catalog, theme):
+    """Intenta conseguir el theme por id o por label."""
+    try:
+        label = catalog.get_theme(identifier=theme)['label']
+    except:
+        label = catalog.get_theme(label=theme)['label']
+
+    label = re.sub(r'[^\wá-úÁ-ÚñÑ .-]+', '',
+                   label, flags=re.UNICODE)
+    return label
+
+
 def convert_iso_string_to_utc(date_string):
     date_time = parser.parse(date_string)
     if date_time.time() == time(0):
diff --git a/pydatajson/federation.py b/pydatajson/federation.py
index 43e932e..b503d95 100644
--- a/pydatajson/federation.py
+++ b/pydatajson/federation.py
@@ -5,11 +5,13 @@ de la API de CKAN.
 """
 
 from __future__ import print_function
+import logging
 from ckanapi import RemoteCKAN
-from ckanapi.errors import NotFound
+from ckanapi.errors import NotFound, NotAuthorized
 from .ckan_utils import map_dataset_to_package, map_theme_to_group
 from .search import get_datasets
 
+logger = logging.getLogger(__name__)
 
 def push_dataset_to_ckan(catalog, owner_org, dataset_origin_identifier,
                          portal_url, apikey, catalog_id=None,
@@ -250,14 +252,20 @@ def harvest_catalog_to_ckan(catalog, portal_url, apikey, catalog_id,
         Returns:
             str: El id del dataset en el catálogo de destino.
     """
-    dataset_list = dataset_list or [ds['identifier']
-                                    for ds in catalog.datasets]
+    # Evitar entrar con valor falsy
+    if dataset_list is None:
+        dataset_list = [ds['identifier'] for ds in catalog.datasets]
     owner_org = owner_org or catalog_id
     harvested = []
     for dataset_id in dataset_list:
-        harvested_id = harvest_dataset_to_ckan(
-            catalog, owner_org, dataset_id, portal_url, apikey, catalog_id)
-        harvested.append(harvested_id)
+        try:
+            harvested_id = harvest_dataset_to_ckan(
+                catalog, owner_org, dataset_id, portal_url, apikey, catalog_id)
+            harvested.append(harvested_id)
+        except (NotAuthorized, NotFound, KeyError, TypeError) as e:
+            logger.error("Error federando catalogo:"+catalog_id+", dataset:"+dataset_id + "al portal: "+portal_url)
+            logger.error(str(e))
+
     return harvested
 
 


Test patch:
diff --git a/tests/test_federation.py b/tests/test_federation.py
index fe95079..9d0515f 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -223,6 +223,13 @@ class PushDatasetTestCase(unittest.TestCase):
             self.assertCountEqual([self.catalog_id+'_'+ds['identifier'] for ds in self.catalog.datasets],
                                   harvested_ids)
 
+    @patch('pydatajson.federation.RemoteCKAN', autospec=True)
+    def test_harvest_catalog_with_empty_list(self, mock_portal):
+        harvested_ids = harvest_catalog_to_ckan(self.catalog, 'portal', 'key', self.catalog_id,
+                                                owner_org='owner', dataset_list=[])
+        mock_portal.assert_not_called()
+        self.assertEqual([], harvested_ids)
+
 
 class RemoveDatasetTestCase(unittest.TestCase):