Repository: ckan/ckan

Issue:
ckan.lib.search.clear is dangerous and should be more explicit.
The call to clear() in ckan.lib.search takes an optional package id.  If no package_id is provided (the default parameter is None) then it will clear the entire search index.

This is likely to make programmer error much more pronounced when they accidentally pass in None, or an empty string and their search index disappears - ask me how I know? 



Discussion:

Resolving patch:
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -222,7 +222,7 @@ def command(self):
                 os.remove(f)
 
             model.repo.clean_db()
-            search.clear()
+            search.clear_all()
             if self.verbose:
                 print 'Cleaning DB: SUCCESS'
         elif cmd == 'upgrade':
@@ -498,9 +498,12 @@ def show(self):
         pprint(index)
 
     def clear(self):
-        from ckan.lib.search import clear
+        from ckan.lib.search import clear, clear_all
         package_id = self.args[1] if len(self.args) > 1 else None
-        clear(package_id)
+        if not package_id:
+            clear_all()
+        else:
+            clear(package_id)
 
     def rebuild_fast(self):
         ###  Get out config but without starting pylons environment ####
diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py
--- a/ckan/lib/search/__init__.py
+++ b/ckan/lib/search/__init__.py
@@ -239,13 +239,16 @@ def show(package_reference):
     return package_query.get_index(package_reference)
 
 
-def clear(package_reference=None):
+def clear(package_reference):
     package_index = index_for(model.Package)
-    if package_reference:
-        log.debug("Clearing search index for dataset %s..." %
-                  package_reference)
-        package_index.delete_package({'id': package_reference})
-    elif not SIMPLE_SEARCH:
+    log.debug("Clearing search index for dataset %s..." %
+              package_reference)
+    package_index.delete_package({'id': package_reference})
+
+
+def clear_all():
+    if not SIMPLE_SEARCH:
+        package_index = index_for(model.Package)
         log.debug("Clearing search index...")
         package_index.clear()
 


Test patch:
diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py
--- a/ckan/tests/helpers.py
+++ b/ckan/tests/helpers.py
@@ -187,7 +187,7 @@ def _apply_config_changes(cls, cfg):
     def setup(self):
         '''Reset the database and clear the search indexes.'''
         reset_db()
-        search.clear()
+        search.clear_all()
 
     @classmethod
     def teardown_class(cls):
diff --git a/ckan/tests/legacy/__init__.py b/ckan/tests/legacy/__init__.py
--- a/ckan/tests/legacy/__init__.py
+++ b/ckan/tests/legacy/__init__.py
@@ -320,7 +320,7 @@ def setup_test_search_index():
     #from ckan import plugins
     if not is_search_supported():
         raise SkipTest("Search not supported")
-    search.clear()
+    search.clear_all()
     #plugins.load('synchronous_search')
 
 def is_search_supported():
diff --git a/ckan/tests/legacy/functional/api/model/test_tag.py b/ckan/tests/legacy/functional/api/model/test_tag.py
--- a/ckan/tests/legacy/functional/api/model/test_tag.py
+++ b/ckan/tests/legacy/functional/api/model/test_tag.py
@@ -1,20 +1,20 @@
 import copy
 
-from nose.tools import assert_equal 
+from nose.tools import assert_equal
 
 from ckan import model
 from ckan.lib.create_test_data import CreateTestData
 import ckan.lib.search as search
 
 from ckan.tests.legacy.functional.api.base import BaseModelApiTestCase
-from ckan.tests.legacy.functional.api.base import Api1TestCase as Version1TestCase 
-from ckan.tests.legacy.functional.api.base import Api2TestCase as Version2TestCase 
+from ckan.tests.legacy.functional.api.base import Api1TestCase as Version1TestCase
+from ckan.tests.legacy.functional.api.base import Api2TestCase as Version2TestCase
 
 class TagsTestCase(BaseModelApiTestCase):
 
     @classmethod
     def setup_class(cls):
-        search.clear()
+        search.clear_all()
         CreateTestData.create()
         cls.testsysadmin = model.User.by_name(u'testsysadmin')
         cls.comment = u'Comment umlaut: \xfc.'
@@ -23,7 +23,7 @@ def setup_class(cls):
 
     @classmethod
     def teardown_class(cls):
-        search.clear()
+        search.clear_all()
         model.repo.rebuild_db()
 
     def test_register_get_ok(self):
@@ -33,7 +33,7 @@ def test_register_get_ok(self):
         assert self.russian.name in results, results
         assert self.tolstoy.name in results, results
         assert self.flexible_tag.name in results, results
-    
+
     def test_entity_get_ok(self):
         offset = self.tag_offset(self.russian.name)
         res = self.app.get(offset, status=self.STATUS_200_OK)
diff --git a/ckan/tests/legacy/functional/api/test_package_search.py b/ckan/tests/legacy/functional/api/test_package_search.py
--- a/ckan/tests/legacy/functional/api/test_package_search.py
+++ b/ckan/tests/legacy/functional/api/test_package_search.py
@@ -34,7 +34,7 @@ def setup_class(self):
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def assert_results(self, res_dict, expected_package_names):
         expected_pkgs = [self.package_ref_from_name(expected_package_name) \
@@ -62,7 +62,7 @@ def check(request_params, expected_params):
 
     def test_00_read_search_params_with_errors(self):
         def check_error(request_params):
-            assert_raises(ValueError, ApiController._get_search_params, request_params)            
+            assert_raises(ValueError, ApiController._get_search_params, request_params)
         # uri json
         check_error(UnicodeMultiDict({'qjson': '{"q": illegal json}'}))
         # posted json
@@ -109,7 +109,7 @@ def test_05_uri_json_tags(self):
         res_dict = self.data_from_res(res)
         self.assert_results(res_dict, [u'annakarenina'])
         assert res_dict['count'] == 1, res_dict
-        
+
     def test_05_uri_json_tags_multiple(self):
         query = {'q': 'tags:russian tags:tolstoy'}
         json_query = self.dumps(query)
@@ -131,7 +131,7 @@ def test_08_uri_qjson_malformed(self):
         offset = self.base_url + '?qjson="q":""' # user forgot the curly braces
         res = self.app.get(offset, status=400)
         self.assert_json_response(res, 'Bad request - Could not read parameters')
-        
+
     def test_09_just_tags(self):
         offset = self.base_url + '?q=tags:russian'
         res = self.app.get(offset, status=200)
@@ -199,7 +199,7 @@ def setup_class(self):
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def test_07_uri_qjson_tags(self):
         query = {'q': '', 'tags':['tolstoy']}
@@ -239,11 +239,11 @@ def test_07_uri_qjson_tags_reverse(self):
         assert res_dict['count'] == 2, res_dict
 
     def test_07_uri_qjson_extras(self):
-        # TODO: solr is not currently set up to allow partial matches 
+        # TODO: solr is not currently set up to allow partial matches
         #       and extras are not saved as multivalued so this
         #       test will fail. Make extras multivalued or remove?
         raise SkipTest()
-    
+
         query = {"geographic_coverage":"England"}
         json_query = self.dumps(query)
         offset = self.base_url + '?qjson=%s' % json_query
@@ -267,7 +267,7 @@ def test_08_all_fields(self):
                               rating=3.0)
         model.Session.add(rating)
         model.repo.commit_and_remove()
-        
+
         query = {'q': 'russian', 'all_fields': 1}
         json_query = self.dumps(query)
         offset = self.base_url + '?qjson=%s' % json_query
diff --git a/ckan/tests/legacy/functional/test_group.py b/ckan/tests/legacy/functional/test_group.py
--- a/ckan/tests/legacy/functional/test_group.py
+++ b/ckan/tests/legacy/functional/test_group.py
@@ -13,7 +13,7 @@ class TestGroup(FunctionalTestCase):
 
     @classmethod
     def setup_class(self):
-        search.clear()
+        search.clear_all()
         model.Session.remove()
         CreateTestData.create()
 
diff --git a/ckan/tests/legacy/lib/test_cli.py b/ckan/tests/legacy/lib/test_cli.py
--- a/ckan/tests/legacy/lib/test_cli.py
+++ b/ckan/tests/legacy/lib/test_cli.py
@@ -8,7 +8,7 @@
 from ckan.lib.create_test_data import CreateTestData
 from ckan.common import json
 
-from ckan.lib.search import index_for,query_for
+from ckan.lib.search import index_for,query_for, clear_all
 
 class TestDb:
     @classmethod
diff --git a/ckan/tests/legacy/lib/test_dictization.py b/ckan/tests/legacy/lib/test_dictization.py
--- a/ckan/tests/legacy/lib/test_dictization.py
+++ b/ckan/tests/legacy/lib/test_dictization.py
@@ -31,7 +31,7 @@ class TestBasicDictize:
     def setup_class(cls):
         # clean the db so we can run these tests on their own
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
         CreateTestData.create()
 
         cls.package_expected = {
diff --git a/ckan/tests/legacy/lib/test_solr_package_search.py b/ckan/tests/legacy/lib/test_solr_package_search.py
--- a/ckan/tests/legacy/lib/test_solr_package_search.py
+++ b/ckan/tests/legacy/lib/test_solr_package_search.py
@@ -56,7 +56,7 @@ def setup_class(cls):
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def _pkg_names(self, result):
         return ' '.join(result['results'])
@@ -316,7 +316,7 @@ def setup_class(cls):
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def test_overall(self):
         check_search_results('annakarenina', 1, ['annakarenina'])
@@ -358,7 +358,7 @@ def setup_class(cls):
     @classmethod
     def teardown_class(self):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def _do_search(self, q, expected_pkgs, count=None):
         query = {
@@ -422,7 +422,7 @@ def setup_class(cls):
     @classmethod
     def teardown_class(self):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def _do_search(self, department, expected_pkgs, count=None):
         result = search.query_for(model.Package).run({'q': 'department: %s' % department})
@@ -467,7 +467,7 @@ def setup_class(cls):
     @classmethod
     def teardown_class(self):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def _do_search(self, q, wanted_results):
         query = {
diff --git a/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py b/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py
--- a/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py
+++ b/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py
@@ -52,19 +52,19 @@ def setup_class(cls):
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
 
     def setup(self):
         self._create_package()
-        
+
     def teardown(self):
         self._remove_package()
         self._remove_package(u'new_name')
-        
+
     def _create_package(self, package=None):
         CreateTestData.create_arbitrary(self.new_pkg_dict)
         return model.Package.by_name(self.new_pkg_dict['name'])
-    
+
     def _remove_package(self, name=None):
         package = model.Package.by_name(name or 'council-owned-litter-bins')
         if package:
@@ -84,7 +84,7 @@ def test_03_update_package_from_dict(self):
         extra = model.PackageExtra(key='published_by', value='barrow')
         package._extras[extra.key] = extra
         model.repo.commit_and_remove()
-        
+
         check_search_results('', 3)
         check_search_results('barrow', 1, ['new_name'])
 
@@ -106,5 +106,5 @@ def test_04_delete_package_from_dict(self):
         rev = model.repo.new_revision()
         package.delete()
         model.repo.commit_and_remove()
-        
+
         check_search_results('', 2)
diff --git a/ckan/tests/legacy/logic/test_action.py b/ckan/tests/legacy/logic/test_action.py
--- a/ckan/tests/legacy/logic/test_action.py
+++ b/ckan/tests/legacy/logic/test_action.py
@@ -35,7 +35,7 @@ class TestAction(WsgiAppCase):
     @classmethod
     def setup_class(cls):
         model.repo.rebuild_db()
-        search.clear()
+        search.clear_all()
         CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
@@ -1349,7 +1349,7 @@ class TestBulkActions(WsgiAppCase):
 
     @classmethod
     def setup_class(cls):
-        search.clear()
+        search.clear_all()
         model.Session.add_all([
             model.User(name=u'sysadmin', apikey=u'sysadmin',
                        password=u'sysadmin', sysadmin=True),
@@ -1436,7 +1436,7 @@ class TestResourceAction(WsgiAppCase):
 
     @classmethod
     def setup_class(cls):
-        search.clear()
+        search.clear_all()
         CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
 
@@ -1539,7 +1539,7 @@ class TestRelatedAction(WsgiAppCase):
 
     @classmethod
     def setup_class(cls):
-        search.clear()
+        search.clear_all()
         CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
 
diff --git a/ckan/tests/legacy/logic/test_tag.py b/ckan/tests/legacy/logic/test_tag.py
--- a/ckan/tests/legacy/logic/test_tag.py
+++ b/ckan/tests/legacy/logic/test_tag.py
@@ -10,7 +10,7 @@
 class TestAction(WsgiAppCase):
     @classmethod
     def setup_class(cls):
-        search.clear()
+        search.clear_all()
         CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
diff --git a/ckan/tests/lib/dictization/test_model_dictize.py b/ckan/tests/lib/dictization/test_model_dictize.py
--- a/ckan/tests/lib/dictization/test_model_dictize.py
+++ b/ckan/tests/lib/dictization/test_model_dictize.py
@@ -14,7 +14,7 @@ class TestGroupListDictize:
 
     def setup(self):
         helpers.reset_db()
-        search.clear()
+        search.clear_all()
 
     def test_group_list_dictize(self):
         group = factories.Group()
@@ -136,7 +136,7 @@ class TestGroupDictize:
 
     def setup(self):
         helpers.reset_db()
-        search.clear()
+        search.clear_all()
 
     def test_group_dictize(self):
         group = factories.Group(name='test_dictize')
diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py
--- a/ckan/tests/logic/action/test_delete.py
+++ b/ckan/tests/logic/action/test_delete.py
@@ -186,7 +186,7 @@ def test_dataset_in_a_purged_group_no_longer_shows_that_group(self):
         assert_equals(dataset_shown['groups'], [])
 
     def test_purged_group_is_not_in_search_results_for_its_ex_dataset(self):
-        search.clear()
+        search.clear_all()
         group = factories.Group()
         dataset = factories.Dataset(groups=[{'name': group['name']}])
 
@@ -288,7 +288,7 @@ def test_dataset_in_a_purged_org_no_longer_shows_that_org(self):
         assert_equals(dataset_shown['owner_org'], None)
 
     def test_purged_org_is_not_in_search_results_for_its_ex_dataset(self):
-        search.clear()
+        search.clear_all()
         org = factories.Organization()
         dataset = factories.Dataset(owner_org=org['id'])
 
@@ -394,7 +394,7 @@ def test_group_no_longer_shows_its_purged_dataset(self):
         assert_equals(dataset_shown['packages'], [])
 
     def test_purged_dataset_is_not_in_search_results(self):
-        search.clear()
+        search.clear_all()
         dataset = factories.Dataset()
 
         def get_search_results():
diff --git a/ckanext/example_idatasetform/tests/test_example_idatasetform.py b/ckanext/example_idatasetform/tests/test_example_idatasetform.py
--- a/ckanext/example_idatasetform/tests/test_example_idatasetform.py
+++ b/ckanext/example_idatasetform/tests/test_example_idatasetform.py
@@ -18,13 +18,13 @@ def setup_class(cls):
 
     def teardown(self):
         model.repo.rebuild_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
     @classmethod
     def teardown_class(cls):
         helpers.reset_db()
         model.repo.rebuild_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
         config.clear()
         config.update(cls.original_config)
@@ -97,7 +97,7 @@ def teardown(self):
     def teardown_class(cls):
         plugins.unload('example_idatasetform_v4')
         helpers.reset_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
         config.clear()
         config.update(cls.original_config)
@@ -139,13 +139,13 @@ def setup_class(cls):
 
     def teardown(self):
         model.repo.rebuild_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
     @classmethod
     def teardown_class(cls):
         plugins.unload('example_idatasetform')
         helpers.reset_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
         config.clear()
         config.update(cls.original_config)
@@ -212,13 +212,13 @@ def setup_class(cls):
 
     def teardown(self):
         model.repo.rebuild_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
     @classmethod
     def teardown_class(cls):
         plugins.unload('example_idatasetform')
         helpers.reset_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
         config.clear()
         config.update(cls.original_config)
diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py
--- a/ckanext/multilingual/tests/test_multilingual_plugin.py
+++ b/ckanext/multilingual/tests/test_multilingual_plugin.py
@@ -58,7 +58,7 @@ def teardown(cls):
         ckan.plugins.unload('multilingual_group')
         ckan.plugins.unload('multilingual_tag')
         ckan.model.repo.rebuild_db()
-        ckan.lib.search.clear()
+        ckan.lib.search.clear_all()
 
     def test_user_read_translation(self):
         '''Test the translation of datasets on user view pages by the