Repository: datosgobar/pydatajson

Issue:
Agregar algunos tests a módulos no testeados
Como `ckan_reader`.

Discussion:

Patch:
diff --git a/pydatajson/ckan_reader.py b/pydatajson/ckan_reader.py
index 88711e7..cb2bf0f 100644
--- a/pydatajson/ckan_reader.py
+++ b/pydatajson/ckan_reader.py
@@ -64,7 +64,7 @@ def read_ckan_catalog(portal_url):
 
             # agrega un nuevo dataset a la lista
             packages.append(portal.call_action(
-                'package_show', {'name_or_id': pkg},
+                'package_show', {'id': pkg},
                 requests_kwargs={"verify": False}
             ))
 
@@ -79,7 +79,7 @@ def read_ckan_catalog(portal_url):
             packages, portal_url)
         catalog["themeTaxonomy"] = map_groups_to_themes(groups)
 
-    except:
+    except Exception as e:
         logger.exception(
             'Error al procesar el portal %s', portal_url, exc_info=True)
 
@@ -281,7 +281,7 @@ Se encontró '%s' como temática global, pero no es mapeable a un
         logger.warn("""
 Se encontraron claves con nombres similares pero no idénticos a "Temática
 global" en 'extras' para el 'package' '%s'. Por favor, considere corregirlas:
-\n%s""", package['name'], almost_accrual)
+\n%s""", package['name'], almost_super_theme)
 
 
 def add_accrualPeriodicity(dataset, package):
@@ -355,6 +355,12 @@ La clave '%s' no está en la metadata del 'resource' '%s'. No
 se puede completar distribution['%s'].""",
                          resource_key, resource['name'], distribution_key)
 
+    if 'attributesDescription' in resource:
+        try:
+            distribution['field'] = json.loads(resource['attributesDescription'])
+        except:
+            logger.exception("Error parseando los fields del resource '%s'", resource['name'])
+
     url_path = ['dataset', resource['package_id'], 'resource', resource['id']]
     distribution["accessURL"] = urljoin(portal_url, "/".join(url_path))
 
@@ -373,7 +379,7 @@ def map_group_to_theme(group):
 
     theme_mapping = {
         'name': 'id',
-        'display_name': 'label',
+        'title': 'label',
         'description': 'description'
     }
 
diff --git a/pydatajson/ckan_utils.py b/pydatajson/ckan_utils.py
index 19bf645..f79c89a 100644
--- a/pydatajson/ckan_utils.py
+++ b/pydatajson/ckan_utils.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 import json
 import re
diff --git a/pydatajson/helpers.py b/pydatajson/helpers.py
index bbff6bc..ff1740d 100644
--- a/pydatajson/helpers.py
+++ b/pydatajson/helpers.py
@@ -15,7 +15,7 @@ import re
 from openpyxl import load_workbook
 from six.moves.urllib_parse import urlparse
 
-from six import string_types
+from six import string_types, iteritems
 from unidecode import unidecode
 
 ABSOLUTE_PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -85,7 +85,7 @@ def parse_date_string(date_string):
 def clean_str(s):
     replacements = {"á": "a", "é": "e", "í": "i", "ó": "o", "ú": "u",
                     ":": "", ".": ""}
-    for old, new in replacements.iteritems():
+    for old, new in iteritems(replacements):
         s = s.replace(old, new)
     return s.lower().strip()
 


Test patch:
diff --git a/tests/test_ckan_reader.py b/tests/test_ckan_reader.py
new file mode 100644
index 0000000..6db249f
--- /dev/null
+++ b/tests/test_ckan_reader.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+try:
+    from mock import patch
+except ImportError:
+    from unittest.mock import patch
+
+from pydatajson.core import DataJson
+from pydatajson.ckan_utils import map_dataset_to_package, map_theme_to_group
+
+from pydatajson.ckan_reader import read_ckan_catalog
+
+SAMPLES_DIR = os.path.join("tests", "samples")
+
+
+def get_sample(sample_filename):
+    return os.path.join(SAMPLES_DIR, sample_filename)
+
+
+class MockPortal(object):
+
+    def __init__(self, sample):
+        self.data_json = DataJson(get_sample(sample))
+
+    def call_action(self, action, data_dict=None, requests_kwargs=None):
+        return getattr(self, action)(data_dict)
+
+    def status_show(self, data_dict):
+        return {'site_title': self.data_json['title'],
+                'site_description': self.data_json['description'],
+                'error_emails_to': self.data_json['publisher']['mbox']}
+
+    def package_list(self, data_dict):
+        return [dataset['title'] for dataset in self.data_json.datasets]
+
+    def group_list(self, data_dict):
+        return [theme['label'] for theme in self.data_json['themeTaxonomy']]
+
+    def package_show(self, data_dict):
+        catalog = self.data_json
+        dataset = catalog.get_dataset(title=data_dict['id'])
+        package = map_dataset_to_package(catalog, dataset, 'owner', demote_superThemes=False, demote_themes=False)
+        for resource in package['resources']:
+            resource['package_id'] = package['id']
+        return package
+
+    def group_show(self, data_dict):
+        catalog = self.data_json
+        theme = catalog.get_theme(label=data_dict['id'])
+        return map_theme_to_group(theme)
+
+
+class CKANReaderTestCase(unittest.TestCase):
+
+    @classmethod
+    @patch('pydatajson.ckan_reader.RemoteCKAN', new=MockPortal)
+    def setUpClass(cls):
+        cls.expected_dj = DataJson(get_sample('full_data.json'))
+        cls.dj = DataJson(read_ckan_catalog('full_data.json'))
+
+    def test_read_ckan_catalog_attributes(self):
+        self.assertEqual(self.expected_dj['title'], self.dj['title'])
+        self.assertEqual(self.expected_dj['description'], self.dj['description'])
+        self.assertEqual(self.expected_dj['superThemeTaxonomy'], self.dj['superThemeTaxonomy'])
+        self.assertEqual(self.expected_dj['publisher']['mbox'], self.dj['publisher']['mbox'])
+
+    def test_read_ckan_catalog_theme_taxonomy(self):
+        self.assertEqual(len(self.expected_dj['themeTaxonomy']),
+                         len(self.dj['themeTaxonomy']))
+        for expected_theme, theme in zip(self.expected_dj['themeTaxonomy'], self.dj['themeTaxonomy']):
+            self.assertDictEqual(expected_theme, theme)
+
+    def test_read_ckan_dataset_attributes(self):
+        self.assertEqual(len(self.expected_dj.datasets),
+                         len(self.dj.datasets))
+        for expected_dataset, dataset in zip(self.expected_dj.datasets, self.dj.datasets):
+            attributes = ['description', 'identifier', 'title', 'landingPage']
+            for attribute in attributes:
+                self.assertEqual(expected_dataset[attribute], dataset[attribute])
+            self.assertDictEqual(expected_dataset['contactPoint'], dataset['contactPoint'])
+            self.assertDictEqual(expected_dataset['publisher'], dataset['publisher'])
+            self.assertSetEqual(set(expected_dataset['keyword']), set(dataset['keyword']))
+
+    def test_read_ckan_distribution_attributes(self):
+        self.assertEqual(len(self.expected_dj.distributions),
+                         len(self.dj.distributions))
+        for expected_dataset, distributions in zip(self.expected_dj.distributions, self.dj.distributions):
+            attributes = ['mediaType', 'byteSize', 'identifier', 'description',
+                          'format', 'title', 'downloadURL']
+            for attribute in attributes:
+                self.assertEqual(expected_dataset[attribute], distributions[attribute])
+
+    def test_read_ckan_field_attributes(self):
+        self.assertEqual(len(self.expected_dj.fields),
+                         len(self.dj.fields))
+        for expected_field, field in zip(self.expected_dj.fields, self.dj.fields):
+            self.assertDictEqual(expected_field, field)
\ No newline at end of file
diff --git a/tests/test_ckan_utils.py b/tests/test_ckan_utils.py
index 66b861a..883dfc7 100644
--- a/tests/test_ckan_utils.py
+++ b/tests/test_ckan_utils.py
@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
 import unittest
 import os
 from .context import pydatajson
 from pydatajson.ckan_utils import *
 from pydatajson.helpers import title_to_name
+
 SAMPLES_DIR = os.path.join("tests", "samples")