Repository: OCHA-DAP/hdx-ckan

Issue:
Refactoring Search Controller
@alexandru-m-g  - breaking count_types into smaller functions, remove redirection package_search.

Marianne - moving package_search into actions



Discussion:
I've made several changes in the refactoring process, including the ones mentioned above. Hopefully didn't introduce too many bugs :).


Resolving patch:
diff --git a/ckanext-hdx_search/ckanext/hdx_search/actions/__init__.py b/ckanext-hdx_search/ckanext/hdx_search/actions/__init__.py
new file mode 100644
diff --git a/ckanext-hdx_search/ckanext/hdx_search/actions/actions.py b/ckanext-hdx_search/ckanext/hdx_search/actions/actions.py
new file mode 100644
--- /dev/null
+++ b/ckanext-hdx_search/ckanext/hdx_search/actions/actions.py
@@ -0,0 +1,189 @@
+import logging
+import copy
+from urllib import urlencode
+
+from pylons import config
+from paste.deploy.converters import asbool
+
+import ckan.logic as logic
+import ckan.lib.base as base
+import ckan.lib.maintain as maintain
+import ckan.lib.navl.dictization_functions as dict_fns
+import ckan.lib.helpers as h
+import ckan.model as model
+import ckan.lib.search as search
+import sqlalchemy
+import ckan.plugins as p
+_validate = dict_fns.validate
+_check_access = logic.check_access
+
+_select = sqlalchemy.sql.select
+_aliased = sqlalchemy.orm.aliased
+_or_ = sqlalchemy.or_
+_and_ = sqlalchemy.and_
+_func = sqlalchemy.func
+_desc = sqlalchemy.desc
+_case = sqlalchemy.case
+_text = sqlalchemy.text
+
+from ckan.common import OrderedDict, _, json, request, c, g, response
+from pydoc_data.topics import topics
+
+log = logging.getLogger(__name__)
+
+render = base.render
+abort = base.abort
+redirect = base.redirect
+
+NotFound = logic.NotFound
+NotAuthorized = logic.NotAuthorized
+ValidationError = logic.ValidationError
+check_access = logic.check_access
+get_action = logic.get_action
+
+def package_search(context, data_dict):
+    '''
+    EDITTED VERSION GETS MORE INFO FROM FACETS
+    '''
+    # sometimes context['schema'] is None
+    schema = (context.get('schema') or
+              logic.schema.default_package_search_schema())
+    data_dict, errors = _validate(data_dict, schema, context)
+    # put the extras back into the data_dict so that the search can
+    # report needless parameters
+    data_dict.update(data_dict.get('__extras', {}))
+    data_dict.pop('__extras', None)
+    if errors:
+        raise ValidationError(errors)
+
+    model = context['model']
+    session = context['session']
+
+    _check_access('package_search', context, data_dict)
+
+    # Move ext_ params to extras and remove them from the root of the search
+    # params, so they don't cause and error
+    data_dict['extras'] = data_dict.get('extras', {})
+    for key in [key for key in data_dict.keys() if key.startswith('ext_')]:
+        data_dict['extras'][key] = data_dict.pop(key)
+
+    # check if some extension needs to modify the search params
+    for item in p.PluginImplementations(p.IPackageController):
+        data_dict = item.before_search(data_dict)
+
+    # the extension may have decided that it is not necessary to perform
+    # the query
+    abort = data_dict.get('abort_search', False)
+
+    if data_dict.get('sort') in (None, 'rank'):
+        data_dict['sort'] = 'score desc, metadata_modified desc'
+
+    results = []
+    if not abort:
+        data_source = 'data_dict' if data_dict.get('use_default_schema',
+                                                   False) else 'validated_data_dict'
+        # return a list of package ids
+        data_dict['fl'] = 'id {0}'.format(data_source)
+
+        # If this query hasn't come from a controller that has set this flag
+        # then we should remove any mention of capacity from the fq and
+        # instead set it to only retrieve public datasets
+        fq = data_dict.get('fq', '')
+        if not context.get('ignore_capacity_check', False):
+            fq = ' '.join(p for p in fq.split(' ')
+                          if not 'capacity:' in p)
+            data_dict['fq'] = fq + ' capacity:"public"'
+
+        # Pop these ones as Solr does not need them
+        extras = data_dict.pop('extras', None)
+
+        query = search.query_for(model.Package)
+        query.run(data_dict)
+
+        # Add them back so extensions can use them on after_search
+        data_dict['extras'] = extras
+
+        for package in query.results:
+            # get the package object
+            package, package_dict = package['id'], package.get(data_source)
+            pkg_query = session.query(model.PackageRevision)\
+                .filter(model.PackageRevision.id == package)\
+                .filter(_and_(
+                    model.PackageRevision.state == u'active',
+                    model.PackageRevision.current == True
+                ))
+            pkg = pkg_query.first()
+
+            # if the index has got a package that is not in ckan then
+            # ignore it.
+            if not pkg:
+                log.warning(
+                    'package %s in index but not in database' % package)
+                continue
+            # use data in search index if there
+            if package_dict:
+                # the package_dict still needs translating when being viewed
+                package_dict = json.loads(package_dict)
+                if context.get('for_view'):
+                    for item in p.PluginImplementations(p.IPackageController):
+                        package_dict = item.before_view(package_dict)
+                results.append(package_dict)
+            else:
+                results.append(model_dictize.package_dictize(pkg, context))
+
+        count = query.count
+        facets = query.facets
+    else:
+        count = 0
+        facets = {}
+        results = []
+
+    search_results = {
+        'count': count,
+        'facets': facets,
+        'results': results,
+        'sort': data_dict['sort']
+    }
+
+    # Transform facets into a more useful data structure.
+    restructured_facets = {}
+    for key, value in facets.items():
+        restructured_facets[key] = {
+            'title': key,
+            'items': []
+        }
+        for key_, value_ in value.items():
+            new_facet_dict = {}
+            new_facet_dict['name'] = key_
+            if key in ('groups', 'organization'):
+                group = model.Group.get(key_)
+                if group:
+                    new_facet_dict['display_name'] = group.display_name
+                    new_facet_dict['description'] = group.description
+                else:
+                    new_facet_dict['display_name'] = key_
+                    new_facet_dict['description'] = ''
+            elif key == 'license_id':
+                license = model.Package.get_license_register().get(key_)
+                if license:
+                    new_facet_dict['display_name'] = license.title
+                else:
+                    new_facet_dict['display_name'] = key_
+            else:
+                new_facet_dict['display_name'] = key_
+            new_facet_dict['count'] = value_
+            restructured_facets[key]['items'].append(new_facet_dict)
+    search_results['search_facets'] = restructured_facets
+
+    # check if some extension needs to modify the search results
+    for item in p.PluginImplementations(p.IPackageController):
+        search_results = item.after_search(search_results, data_dict)
+
+    # After extensions have had a chance to modify the facets, sort them by
+    # display name.
+    for facet in search_results['search_facets']:
+        search_results['search_facets'][facet]['items'] = sorted(
+            search_results['search_facets'][facet]['items'],
+            key=lambda facet: facet['display_name'], reverse=True)
+
+    return search_results
diff --git a/ckanext-hdx_search/ckanext/hdx_search/controllers/search_controller.py b/ckanext-hdx_search/ckanext/hdx_search/controllers/search_controller.py
--- a/ckanext-hdx_search/ckanext/hdx_search/controllers/search_controller.py
+++ b/ckanext-hdx_search/ckanext/hdx_search/controllers/search_controller.py
@@ -46,7 +46,7 @@
 LARGE_NUM_OF_ITEMS = 25
 
 from ckan.controllers.package import PackageController
-
+import ckanext.hdx_search.actions.actions as hdx_actions
 
 def _encode_params(params):
     return [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
@@ -109,155 +109,6 @@ def extract_one_indicator(result):
         indicator = None
     return indicator
 
-
-def package_search(context, data_dict):
-    '''
-    EDITTED VERSION GETS MORE INFO FROM FACETS
-    '''
-    # sometimes context['schema'] is None
-    schema = (context.get('schema') or
-              logic.schema.default_package_search_schema())
-    data_dict, errors = _validate(data_dict, schema, context)
-    # put the extras back into the data_dict so that the search can
-    # report needless parameters
-    data_dict.update(data_dict.get('__extras', {}))
-    data_dict.pop('__extras', None)
-    if errors:
-        raise ValidationError(errors)
-
-    model = context['model']
-    session = context['session']
-
-    _check_access('package_search', context, data_dict)
-
-    # Move ext_ params to extras and remove them from the root of the search
-    # params, so they don't cause and error
-    data_dict['extras'] = data_dict.get('extras', {})
-    for key in [key for key in data_dict.keys() if key.startswith('ext_')]:
-        data_dict['extras'][key] = data_dict.pop(key)
-
-    # check if some extension needs to modify the search params
-    for item in p.PluginImplementations(p.IPackageController):
-        data_dict = item.before_search(data_dict)
-
-    # the extension may have decided that it is not necessary to perform
-    # the query
-    abort = data_dict.get('abort_search', False)
-
-    if data_dict.get('sort') in (None, 'rank'):
-        data_dict['sort'] = 'score desc, metadata_modified desc'
-
-    results = []
-    if not abort:
-        data_source = 'data_dict' if data_dict.get('use_default_schema',
-                                                   False) else 'validated_data_dict'
-        # return a list of package ids
-        data_dict['fl'] = 'id {0}'.format(data_source)
-
-        # If this query hasn't come from a controller that has set this flag
-        # then we should remove any mention of capacity from the fq and
-        # instead set it to only retrieve public datasets
-        fq = data_dict.get('fq', '')
-        if not context.get('ignore_capacity_check', False):
-            fq = ' '.join(p for p in fq.split(' ')
-                          if not 'capacity:' in p)
-            data_dict['fq'] = fq + ' capacity:"public"'
-
-        # Pop these ones as Solr does not need them
-        extras = data_dict.pop('extras', None)
-
-        query = search.query_for(model.Package)
-        query.run(data_dict)
-
-        # Add them back so extensions can use them on after_search
-        data_dict['extras'] = extras
-
-        for package in query.results:
-            # get the package object
-            package, package_dict = package['id'], package.get(data_source)
-            pkg_query = session.query(model.PackageRevision)\
-                .filter(model.PackageRevision.id == package)\
-                .filter(_and_(
-                    model.PackageRevision.state == u'active',
-                    model.PackageRevision.current == True
-                ))
-            pkg = pkg_query.first()
-
-            # if the index has got a package that is not in ckan then
-            # ignore it.
-            if not pkg:
-                log.warning(
-                    'package %s in index but not in database' % package)
-                continue
-            # use data in search index if there
-            if package_dict:
-                # the package_dict still needs translating when being viewed
-                package_dict = json.loads(package_dict)
-                if context.get('for_view'):
-                    for item in p.PluginImplementations(p.IPackageController):
-                        package_dict = item.before_view(package_dict)
-                results.append(package_dict)
-            else:
-                results.append(model_dictize.package_dictize(pkg, context))
-
-        count = query.count
-        facets = query.facets
-    else:
-        count = 0
-        facets = {}
-        results = []
-
-    search_results = {
-        'count': count,
-        'facets': facets,
-        'results': results,
-        'sort': data_dict['sort']
-    }
-
-    # Transform facets into a more useful data structure.
-    restructured_facets = {}
-    for key, value in facets.items():
-        restructured_facets[key] = {
-            'title': key,
-            'items': []
-        }
-        for key_, value_ in value.items():
-            new_facet_dict = {}
-            new_facet_dict['name'] = key_
-            if key in ('groups', 'organization'):
-                group = model.Group.get(key_)
-                if group:
-                    new_facet_dict['display_name'] = group.display_name
-                    new_facet_dict['description'] = group.description
-                else:
-                    new_facet_dict['display_name'] = key_
-                    new_facet_dict['description'] = ''
-            elif key == 'license_id':
-                license = model.Package.get_license_register().get(key_)
-                if license:
-                    new_facet_dict['display_name'] = license.title
-                else:
-                    new_facet_dict['display_name'] = key_
-            else:
-                new_facet_dict['display_name'] = key_
-            new_facet_dict['count'] = value_
-            restructured_facets[key]['items'].append(new_facet_dict)
-    search_results['search_facets'] = restructured_facets
-
-    # check if some extension needs to modify the search results
-    for item in p.PluginImplementations(p.IPackageController):
-        search_results = item.after_search(search_results, data_dict)
-
-    # After extensions have had a chance to modify the facets, sort them by
-    # display name.
-    for facet in search_results['search_facets']:
-        search_results['search_facets'][facet]['items'] = sorted(
-            search_results['search_facets'][facet]['items'],
-            key=lambda facet: facet['display_name'], reverse=True)
-
-    return search_results
-
-
 def sort_features(features):
     return sorted(features, key=lambda x: x['count'], reverse=True)
 
@@ -499,7 +350,7 @@ def _performing_search(self, q, fq, facets, limit, page, sort_by,
 
         self._decide_adding_dataset_criteria(data_dict)
 
-        query = package_search(context, data_dict)
+        query = hdx_actions.package_search(context, data_dict)
 
         all_result = search_for_all(context, data_dict)
         c.dataset_counts, c.indicator_counts = extract_counts(all_result)


Test patch: