Repository: ckan/ckan

Issue:
In datastore_search, parameter include_total=False doesn't work
### CKAN Version if known (or site URL)
master

### Please describe the expected behaviour
When calling action function `datastore_search` you can set parameter include_total=False when you don't need the count. This should prevent an SQL query e.g. `SELECT  count(*) FROM "47e77d3c-6c77-44f2-bbf4-740b1b28bf5f"  ;` Counting rows is notoriously expensive, so we should avoid this when possible.

### Please describe the actual behaviour
The parameter didn't change behaviour. The `SELECT count(*)...` occurred anyway. This causes the resource page to display slowly.

This is not normally a problem, but when the datastore db is being taxed (e.g. by ckan-xloader loading in data) the resource page for large datasets [can become unresponsive](https://github.com/ckan/ckanext-xloader/issues/41). So it is important to save on these expensive requests where we can.

### What steps can be taken to reproduce the issue?
You can see the `SELECT count(*) ...` occurring by turning on sqlalchemy logging, viewing a resource page (e.g. `/dataset/abc/resource/xyz` for a CSV that has been xloaded/datapushed.

If you put a pdb breakpoint in `ckan/ckanext/datastore/helpers.py:datastore_dictionary()` you see it calls datastore_search with include_total=False, however in datastore_search you see that gets changed to True when stepping past this line:
```
    data_dict, errors = _validate(data_dict, schema, context)
```


Discussion:

Resolving patch:
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -2280,7 +2280,8 @@ def resource_view_get_fields(resource):
 
     data = {
         'resource_id': resource['id'],
-        'limit': 0
+        'limit': 0,
+        'include_total': False,
     }
     result = logic.get_action('datastore_search')({}, data)
 
diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py
--- a/ckan/lib/navl/validators.py
+++ b/ckan/lib/navl/validators.py
@@ -74,11 +74,13 @@ def ignore(key, data, errors, context):
     raise StopOnError
 
 def default(default_value):
+    '''When key is missing or value is an empty string or None, replace it with
+    a default value'''
 
     def callable(key, data, errors, context):
 
         value = data.get(key)
-        if not value or value is missing:
+        if value is None or value == '' or value is missing:
             data[key] = default_value
 
     return callable
diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py
--- a/ckanext/datastore/controller.py
+++ b/ckanext/datastore/controller.py
@@ -143,7 +143,7 @@ def result_page(offs, lim):
             'offset': offs,
             'sort': '_id',
             'records_format': records_format,
-            'include_total': 'false',  # XXX: default() is broken
+            'include_total': False,
         })
 
     result = result_page(offset, limit)


Test patch:
diff --git a/ckan/tests/legacy/lib/test_navl.py b/ckan/tests/legacy/lib/test_navl.py
--- a/ckan/tests/legacy/lib/test_navl.py
+++ b/ckan/tests/legacy/lib/test_navl.py
@@ -186,21 +186,6 @@ def test_basic_errors():
 
     assert errors == {('__junk',): [u"The input field [('4', 1, '30')] was not expected."], ('1',): [u'Missing value'], ('__extras',): [u'The input field __extras was not expected.']}, errors
 
-def test_default():
-    schema = {
-        "__junk": [ignore],
-        "__extras": [ignore, default("weee")],
-        "__before": [ignore],
-        "__after": [ignore],
-        "0": [default("default")],
-        "1": [default("default")],
-    }
-
-    converted_data, errors = validate_flattened(data, schema)
-
-    assert not errors
-    assert converted_data == {('1',): 'default', ('0',): '0 value'}, converted_data
-
 
 def test_flatten():
 
diff --git a/ckan/tests/lib/navl/test_validators.py b/ckan/tests/lib/navl/test_validators.py
--- a/ckan/tests/lib/navl/test_validators.py
+++ b/ckan/tests/lib/navl/test_validators.py
@@ -8,6 +8,10 @@
 import nose.tools
 
 import ckan.tests.factories as factories
+import ckan.lib.navl.validators as validators
+
+
+eq_ = nose.tools.eq_
 
 
 def returns_None(function):
@@ -222,7 +226,6 @@ def test_ignore_missing_with_value_missing(self):
 
         '''
         import ckan.lib.navl.dictization_functions as df
-        import ckan.lib.navl.validators as validators
 
         for value in (None, df.missing, 'skip'):
 
@@ -251,8 +254,6 @@ def test_ignore_missing_with_a_value(self):
         nothing.
 
         '''
-        import ckan.lib.navl.validators as validators
-
         key = ('key to be validated',)
         data = factories.validator_data_dict()
         data[key] = 'value to be validated'
@@ -265,3 +266,32 @@ def test_ignore_missing_with_a_value(self):
         def call_validator(*args, **kwargs):
             return validators.ignore_missing(*args, **kwargs)
         call_validator(key=key, data=data, errors=errors, context={})
+
+
+class TestDefault(object):
+    def test_key_doesnt_exist(self):
+        dict_ = {}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'default_value'})
+
+    def test_value_is_none(self):
+        dict_ = {'key': None}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'default_value'})
+
+    def test_value_is_empty_string(self):
+        dict_ = {'key': ''}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'default_value'})
+
+    def test_value_is_false(self):
+        # False is a consciously set value, so should not be changed to the
+        # default
+        dict_ = {'key': False}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': False})
+
+    def test_already_has_a_value(self):
+        dict_ = {'key': 'original'}
+        validators.default('default_value')('key', dict_, {}, {})
+        eq_(dict_, {'key': 'original'})
diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py
--- a/ckanext/datastore/tests/test_search.py
+++ b/ckanext/datastore/tests/test_search.py
@@ -100,6 +100,23 @@ def test_all_params_work_with_fields_with_whitespaces(self):
         result_years = [r['the year'] for r in result['records']]
         assert_equals(result_years, [2013])
 
+    def test_search_without_total(self):
+        resource = factories.Resource()
+        data = {
+            'resource_id': resource['id'],
+            'force': True,
+            'records': [
+                {'the year': 2014},
+                {'the year': 2013},
+            ],
+        }
+        result = helpers.call_action('datastore_create', **data)
+        search_data = {
+            'resource_id': resource['id'],
+            'include_total': False
+        }
+        result = helpers.call_action('datastore_search', **search_data)
+        assert 'total' not in result
 
 
 class TestDatastoreSearch(DatastoreLegacyTestBase):