Repository: ckan/ckan

Issue:
DataPusher called multiple times when creating a dataset
When you upload a CSV the DataPusher [gets called](https://github.com/ckan/ckan/blob/master/ckanext/datapusher/plugin.py#L98) to do its thing.
On https://github.com/ckan/ckan/pull/2234 we added a couple of calls to `resource_patch` on `datastore_create` and `datastore_delete` to set the `datastore_active` extra on the resource.

This is all very well but this update on the resource triggers again the DataPusher, which pings `datastore_delete` and `datastore_create`, which trigger...

Not sure how we can handle this on the `notify` extension point, as we only get the model object there.



Discussion:
And for the record, before 2.5, DataPusher always got called 2 times when creating a dataset because the wonderful [`new_resource`](https://github.com/ckan/ckan/blob/master/ckan/controllers/package.py#L608) action from the package controller first calls `resource_create` and then `package_update`.

Hi, may i know is the problem solve the ? My ckan is install on ubuntu 12.10 with ckan version 2.5.1 
I have successfully install datapusher and datastore. But something when i upload a dataset, datapusher always got called 2 times. This error happened randomly, sometime it push normally but most of the time it pushes twice.

Just like amercader said, I check the log, most of the api got call twice when i upload a dataset. Such as store_create,datastore_delete, datastore_search etc. But teach me if there is any solution for this. Thank you

@TkTech I'm working on a patch for this, so grabbing it from you


Resolving patch:
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -3,6 +3,8 @@
 import urlparse
 import datetime
 
+from dateutil.parser import parse as parse_date
+
 import pylons
 import requests
 
@@ -37,7 +39,6 @@ def datapusher_submit(context, data_dict):
 
     :rtype: bool
     '''
-
     schema = context.get('schema', dpschema.datapusher_submit_schema())
     data_dict, errors = _validate(data_dict, schema, context)
     if errors:
@@ -47,6 +48,13 @@ def datapusher_submit(context, data_dict):
 
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
+    try:
+        resource_dict = p.toolkit.get_action('resource_show')(context, {
+            'id': res_id,
+        })
+    except logic.NotFound:
+        return False
+
     datapusher_url = pylons.config.get('ckan.datapusher.url')
 
     site_url = pylons.config['ckan.site_url']
@@ -100,7 +108,9 @@ def datapusher_submit(context, data_dict):
                     'ignore_hash': data_dict.get('ignore_hash', False),
                     'ckan_url': site_url,
                     'resource_id': res_id,
-                    'set_url_type': data_dict.get('set_url_type', False)
+                    'set_url_type': data_dict.get('set_url_type', False),
+                    'task_created': task['last_updated'],
+                    'original_url': resource_dict.get('url'),
                 }
             }))
         r.raise_for_status()
@@ -166,6 +176,9 @@ def datapusher_hook(context, data_dict):
 
     task['state'] = status
     task['last_updated'] = str(datetime.datetime.now())
+
+    resubmit = False
+
     if status == 'complete':
         # Create default views for resource if necessary (only the ones that
         # require data to be in the DataStore)
@@ -186,9 +199,36 @@ def datapusher_hook(context, data_dict):
                 'create_datastore_views': True,
             })
 
+        # Check if the uploaded file has been modified in the meantime
+        if (resource_dict.get('last_modified') and
+                metadata.get('task_created')):
+            try:
+                last_modified_datetime = parse_date(
+                    resource_dict['last_modified'])
+                task_created_datetime = parse_date(metadata['task_created'])
+                if last_modified_datetime > task_created_datetime:
+                    log.debug('Uploaded file more recent: {0} > {1}'.format(
+                        last_modified_datetime, task_created_datetime))
+                    resubmit = True
+            except ValueError:
+                pass
+        # Check if the URL of the file has been modified in the meantime
+        elif (resource_dict.get('url') and
+                metadata.get('original_url') and
+                resource_dict['url'] != metadata['original_url']):
+            log.debug('URLs are different: {0} != {1}'.format(
+                resource_dict['url'], metadata['original_url']))
+            resubmit = True
+
     context['ignore_auth'] = True
     p.toolkit.get_action('task_status_update')(context, task)
 
+    if resubmit:
+        log.debug('Resource {0} has been modified, '
+                  'resubmitting to DataPusher'.format(res_id))
+        p.toolkit.get_action('datapusher_submit')(
+            context, {'resource_id': res_id})
+
 
 def datapusher_status(context, data_dict):
     ''' Get the status of a datapusher job for a certain resource.
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -109,6 +109,25 @@ def notify(self, entity, operation=None):
                         entity.url_type != 'datapusher'):
 
                     try:
+                        task = p.toolkit.get_action('task_status_show')(
+                            context, {
+                                'entity_id': entity.id,
+                                'task_type': 'datapusher',
+                                'key': 'datapusher'}
+                        )
+                        if task.get('state') == 'pending':
+                            # There already is a pending DataPusher submission,
+                            # skip this one ...
+                            log.debug(
+                                'Skipping DataPusher submission for '
+                                'resource {0}'.format(entity.id))
+                            return
+                    except p.toolkit.ObjectNotFound:
+                        pass
+
+                    try:
+                        log.debug('Submitting resource {0}'.format(entity.id) +
+                                  ' to DataPusher')
                         p.toolkit.get_action('datapusher_submit')(context, {
                             'resource_id': entity.id
                         })
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -143,8 +143,13 @@ def datastore_create(context, data_dict):
         raise p.toolkit.ValidationError(str(err))
 
     # Set the datastore_active flag on the resource if necessary
-    p.toolkit.get_action('resource_patch')(
-        context, {'id': data_dict['resource_id'], 'datastore_active': True})
+    if resource.extras.get('datastore_active') is not True:
+        log.debug(
+            'Setting datastore_active=True on resource {0}'.format(resource.id)
+        )
+        p.toolkit.get_action('resource_patch')(
+            context,
+            {'id': data_dict['resource_id'], 'datastore_active': True})
 
     result.pop('id', None)
     result.pop('private', None)
@@ -345,9 +350,17 @@ def datastore_delete(context, data_dict):
     result = db.delete(context, data_dict)
 
     # Set the datastore_active flag on the resource if necessary
-    if not data_dict.get('filters'):
+    model = _get_or_bust(context, 'model')
+    resource = model.Resource.get(data_dict['resource_id'])
+
+    if (not data_dict.get('filters') and
+            resource.extras.get('datastore_active') is True):
+        log.debug(
+            'Setting datastore_active=True on resource {0}'.format(resource.id)
+        )
         p.toolkit.get_action('resource_patch')(
-            context, {'id': data_dict['resource_id'], 'datastore_active': False})
+            context, {'id': data_dict['resource_id'],
+                      'datastore_active': False})
 
     result.pop('id', None)
     result.pop('connection_url')


Test patch:
diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py
--- a/ckan/tests/helpers.py
+++ b/ckan/tests/helpers.py
@@ -20,6 +20,7 @@
 import webtest
 from pylons import config
 import nose.tools
+import mock
 
 import ckan.lib.search as search
 import ckan.config.middleware
@@ -320,3 +321,60 @@ def wrapper(*args, **kwargs):
             return return_value
         return nose.tools.make_decorator(func)(wrapper)
     return decorator
+
+
+def mock_action(action_name):
+    '''
+    Decorator to easily mock a CKAN action in the context of a test function
+
+    It adds a mock object for the provided action as a parameter to the test
+    function. The mock is discarded at the end of the function, even if there
+    is an exception raised.
+
+    Note that this mocks the action both when it's called directly via
+    ``ckan.logic.get_action`` and via ``ckan.plugins.toolkit.get_action``.
+
+    Usage::
+
+        @mock_action('user_list')
+        def test_mock_user_list(self, mock_user_list):
+
+            mock_user_list.return_value = 'hi'
+
+            # user_list is mocked
+            eq_(helpers.call_action('user_list', {}), 'hi')
+
+            assert mock_user_list.called
+
+    :param action_name: the name of the action to be mocked,
+        e.g. ``package_create``
+    :type action_name: string
+
+    '''
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            mock_action = mock.MagicMock()
+
+            from ckan.logic import get_action as original_get_action
+
+            def side_effect(called_action_name):
+                if called_action_name == action_name:
+                    return mock_action
+                else:
+                    return original_get_action(called_action_name)
+            try:
+                with mock.patch('ckan.logic.get_action') as mock_get_action, \
+                        mock.patch('ckan.plugins.toolkit.get_action') \
+                        as mock_get_action_toolkit:
+                    mock_get_action.side_effect = side_effect
+                    mock_get_action_toolkit.side_effect = side_effect
+
+                    new_args = args + tuple([mock_action])
+                    return_value = func(*new_args, **kwargs)
+            finally:
+                # Make sure to stop the mock, even with an exception
+                mock_action.stop()
+            return return_value
+
+        return nose.tools.make_decorator(func)(wrapper)
+    return decorator
diff --git a/ckanext/datapusher/tests/test_action.py b/ckanext/datapusher/tests/test_action.py
new file mode 100644
--- /dev/null
+++ b/ckanext/datapusher/tests/test_action.py
@@ -0,0 +1,277 @@
+import datetime
+
+from nose.tools import eq_
+
+import ckan.plugins as p
+from ckan.tests import helpers, factories
+
+
+class TestDataPusherAction(object):
+
+    @classmethod
+    def setup_class(cls):
+        if not p.plugin_loaded('datastore'):
+            p.load('datastore')
+        if not p.plugin_loaded('datapusher'):
+            p.load('datapusher')
+
+        helpers.reset_db()
+
+    @classmethod
+    def teardown_class(cls):
+
+        p.unload('datapusher')
+        p.unload('datastore')
+
+        helpers.reset_db()
+
+    def _pending_task(self, resource_id):
+        return {
+            'entity_id': resource_id,
+            'entity_type': 'resource',
+            'task_type': 'datapusher',
+            'last_updated': str(datetime.datetime.now()),
+            'state': 'pending',
+            'key': 'datapusher',
+            'value': '{}',
+            'error': '{}',
+        }
+
+    @helpers.mock_action('datapusher_submit')
+    def test_submit(self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        assert not mock_datapusher_submit.called
+
+        helpers.call_action('resource_create', {},
+                            package_id=dataset['id'],
+                            url='http://example.com/file.csv',
+                            format='CSV')
+
+        assert mock_datapusher_submit.called
+
+    @helpers.mock_action('datapusher_submit')
+    def test_submit_when_url_changes(self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action('resource_create', {},
+                                       package_id=dataset['id'],
+                                       url='http://example.com/file.pdf',
+                                       )
+
+        assert not mock_datapusher_submit.called
+
+        helpers.call_action('resource_update', {},
+                            id=resource['id'],
+                            package_id=dataset['id'],
+                            url='http://example.com/file.csv',
+                            format='CSV'
+                            )
+
+        assert mock_datapusher_submit.called
+
+    @helpers.mock_action('datapusher_submit')
+    def test_does_not_submit_while_ongoing_job(self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action('resource_create', {},
+                                       package_id=dataset['id'],
+                                       url='http://example.com/file.CSV',
+                                       format='CSV'
+                                       )
+
+        assert mock_datapusher_submit.called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Create a task with a state pending to mimic an ongoing job
+        # on the DataPusher
+        helpers.call_action('task_status_update', {},
+                            **self._pending_task(resource['id']))
+
+        # Update the resource
+        helpers.call_action('resource_update', {},
+                            id=resource['id'],
+                            package_id=dataset['id'],
+                            url='http://example.com/file.csv',
+                            format='CSV',
+                            description='Test',
+                            )
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+    @helpers.mock_action('datapusher_submit')
+    def test_resubmits_if_url_changes_in_the_meantime(
+            self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action('resource_create', {},
+                                       package_id=dataset['id'],
+                                       url='http://example.com/file.csv',
+                                       format='CSV'
+                                       )
+
+        assert mock_datapusher_submit.called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Create a task with a state pending to mimic an ongoing job
+        # on the DataPusher
+        task = helpers.call_action('task_status_update', {},
+                                   **self._pending_task(resource['id']))
+
+        # Update the resource, set a new URL
+        helpers.call_action('resource_update', {},
+                            id=resource['id'],
+                            package_id=dataset['id'],
+                            url='http://example.com/another.file.csv',
+                            format='CSV',
+                            )
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Call datapusher_hook with state complete, to mock the DataPusher
+        # finishing the job and telling CKAN
+        data_dict = {
+            'metadata': {
+                'resource_id': resource['id'],
+                'original_url': 'http://example.com/file.csv',
+                'task_created': task['last_updated'],
+            },
+            'status': 'complete',
+        }
+        helpers.call_action('datapusher_hook', {}, **data_dict)
+
+        # datapusher_submit was called again
+        eq_(len(mock_datapusher_submit.mock_calls), 2)
+
+    @helpers.mock_action('datapusher_submit')
+    def test_resubmits_if_upload_changes_in_the_meantime(
+            self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action('resource_create', {},
+                                       package_id=dataset['id'],
+                                       url='http://example.com/file.csv',
+                                       format='CSV'
+                                       )
+
+        assert mock_datapusher_submit.called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Create a task with a state pending to mimic an ongoing job
+        # on the DataPusher
+        task = helpers.call_action('task_status_update', {},
+                                   **self._pending_task(resource['id']))
+
+        # Update the resource, set a new last_modified (changes on file upload)
+        helpers.call_action('resource_update', {},
+                            id=resource['id'],
+                            package_id=dataset['id'],
+                            url='http://example.com/file.csv',
+                            format='CSV',
+                            last_modified=datetime.datetime.now().isoformat()
+                            )
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Call datapusher_hook with state complete, to mock the DataPusher
+        # finishing the job and telling CKAN
+        data_dict = {
+            'metadata': {
+                'resource_id': resource['id'],
+                'original_url': 'http://example.com/file.csv',
+                'task_created': task['last_updated'],
+            },
+            'status': 'complete',
+        }
+        helpers.call_action('datapusher_hook', {}, **data_dict)
+
+        # datapusher_submit was called again
+        eq_(len(mock_datapusher_submit.mock_calls), 2)
+
+    @helpers.mock_action('datapusher_submit')
+    def test_does_not_resubmit_if_a_resource_field_changes_in_the_meantime(
+            self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action('resource_create', {},
+                                       package_id=dataset['id'],
+                                       url='http://example.com/file.csv',
+                                       format='CSV'
+                                       )
+
+        assert mock_datapusher_submit.called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Create a task with a state pending to mimic an ongoing job
+        # on the DataPusher
+        task = helpers.call_action('task_status_update', {},
+                                   **self._pending_task(resource['id']))
+
+        # Update the resource, set a new description
+        helpers.call_action('resource_update', {},
+                            id=resource['id'],
+                            package_id=dataset['id'],
+                            url='http://example.com/file.csv',
+                            format='CSV',
+                            description='Test',
+                            )
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Call datapusher_hook with state complete, to mock the DataPusher
+        # finishing the job and telling CKAN
+        data_dict = {
+            'metadata': {
+                'resource_id': resource['id'],
+                'original_url': 'http://example.com/file.csv',
+                'task_created': task['last_updated'],
+            },
+            'status': 'complete',
+        }
+        helpers.call_action('datapusher_hook', {}, **data_dict)
+
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+    @helpers.mock_action('datapusher_submit')
+    def test_does_not_resubmit_if_a_dataset_field_changes_in_the_meantime(
+            self, mock_datapusher_submit):
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action('resource_create', {},
+                                       package_id=dataset['id'],
+                                       url='http://example.com/file.csv',
+                                       format='CSV'
+                                       )
+
+        assert mock_datapusher_submit.called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Create a task with a state pending to mimic an ongoing job
+        # on the DataPusher
+        task = helpers.call_action('task_status_update', {},
+                                   **self._pending_task(resource['id']))
+
+        # Update the parent dataset
+        helpers.call_action('package_update', {},
+                            id=dataset['id'],
+                            notes='Test notes',
+                            resources=[resource]
+                            )
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
+
+        # Call datapusher_hook with state complete, to mock the DataPusher
+        # finishing the job and telling CKAN
+        data_dict = {
+            'metadata': {
+                'resource_id': resource['id'],
+                'original_url': 'http://example.com/file.csv',
+                'task_created': task['last_updated'],
+            },
+            'status': 'complete',
+        }
+        helpers.call_action('datapusher_hook', {}, **data_dict)
+
+        # Not called
+        eq_(len(mock_datapusher_submit.mock_calls), 1)
diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
--- a/ckanext/datastore/tests/test_create.py
+++ b/ckanext/datastore/tests/test_create.py
@@ -189,6 +189,47 @@ def _execute_sql(self, sql, *args):
         session = orm.scoped_session(orm.sessionmaker(bind=engine))
         return session.connection().execute(sql, *args)
 
+    def test_sets_datastore_active_on_resource_on_create(self):
+        resource = factories.Resource()
+
+        assert_equal(resource['datastore_active'], False)
+
+        data = {
+            'resource_id': resource['id'],
+            'force': True,
+            'records': [
+                {'book': 'annakarenina', 'author': 'tolstoy'}
+            ]
+        }
+
+        helpers.call_action('datastore_create', **data)
+
+        resource = helpers.call_action('resource_show', id=resource['id'])
+
+        assert_equal(resource['datastore_active'], True)
+
+    def test_sets_datastore_active_on_resource_on_delete(self):
+        resource = factories.Resource(datastore_active=True)
+
+        assert_equal(resource['datastore_active'], True)
+
+        data = {
+            'resource_id': resource['id'],
+            'force': True,
+            'records': [
+                {'book': 'annakarenina', 'author': 'tolstoy'}
+            ]
+        }
+
+        helpers.call_action('datastore_create', **data)
+
+        helpers.call_action('datastore_delete', resource_id=resource['id'],
+                            force=True)
+
+        resource = helpers.call_action('resource_show', id=resource['id'])
+
+        assert_equal(resource['datastore_active'], False)
+
 
 class TestDatastoreCreate(tests.WsgiAppCase):
     sysadmin_user = None