Repository: OCHA-DAP/hdx-ckan

Issue:
Server Logs: google bot accessing a resource
Below is the full error we are getting in logs. It might have two parts:
1. An exception that must be catched and nicer displayed
2. we must inform google that the resource has another url. Do we want nicer url?
For example the url google is trying to access: http://data.hdx.rwlabs.org/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx
is changed to this one:
http://data.hdx.rwlabs.org/storage/f/2014-11-07T18%3A43%3A40.574Z/health-care-workers-deaths.xlsx

```
URL: http://data.hdx.rwlabs.org/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx
Module weberror.errormiddleware:162 in __call__
<<              __traceback_supplement__ = Supplement, self, environ
                   sr_checker = ResponseStartChecker(start_response)
                   app_iter = self.application(environ, sr_checker)
                   return self.make_catching_iter(app_iter, environ, sr_checker)
               except:
>>  app_iter = self.application(environ, sr_checker)
Module webob.dec:147 in __call__
<<                  if self.middleware_wraps:
                           args = (self.middleware_wraps,) + args
                       resp = self.call_func(req, *args, **self.kwargs)
                   except webob.exc.HTTPException, resp:
                       pass
>>  resp = self.call_func(req, *args, **self.kwargs)
Module webob.dec:208 in call_func
<<          """Call the wrapped function; override this in a subclass to
               change how the function is called."""
               return self.func(req, *args, **kwargs)

           def clone(self, func=None, **kw):
>>  return self.func(req, *args, **kwargs)
Module fanstatic.publisher:234 in __call__
<<              # the trigger segment is not in the URL, so we delegate
                   # to the original application
                   return request.get_response(self.app)
               # the trigger is in there, so let whatever is behind the
               # trigger be handled by the publisher
>>  return request.get_response(self.app)
Module webob.request:1053 in get_response
<<          else:
                   status, headers, app_iter = self.call_application(
                       application, catch_exc_info=False)
               return self.ResponseClass(
                   status=status, headerlist=list(headers), app_iter=app_iter,
>>  application, catch_exc_info=False)
Module webob.request:1022 in call_application
<<              captured[:] = [status, headers, exc_info]
                   return output.append
               app_iter = application(self.environ, start_response)
               if output or not captured:
                   try:
>>  app_iter = application(self.environ, start_response)
Module webob.dec:147 in __call__
<<                  if self.middleware_wraps:
                           args = (self.middleware_wraps,) + args
                       resp = self.call_func(req, *args, **self.kwargs)
                   except webob.exc.HTTPException, resp:
                       pass
>>  resp = self.call_func(req, *args, **self.kwargs)
Module webob.dec:208 in call_func
<<          """Call the wrapped function; override this in a subclass to
               change how the function is called."""
               return self.func(req, *args, **kwargs)

           def clone(self, func=None, **kw):
>>  return self.func(req, *args, **kwargs)
Module fanstatic.injector:54 in __call__
<<          # Get the response from the wrapped application:
               response = request.get_response(self.app)

               # We only continue if the content-type is appropriate.
>>  response = request.get_response(self.app)
Module webob.request:1053 in get_response
<<          else:
                   status, headers, app_iter = self.call_application(
                       application, catch_exc_info=False)
               return self.ResponseClass(
                   status=status, headerlist=list(headers), app_iter=app_iter,
>>  application, catch_exc_info=False)
Module webob.request:1022 in call_application
<<              captured[:] = [status, headers, exc_info]
                   return output.append
               app_iter = application(self.environ, start_response)
               if output or not captured:
                   try:
>>  app_iter = application(self.environ, start_response)
Module beaker.middleware:73 in __call__
<<                                                     self.cache_manager)
               environ[self.environ_key] = self.cache_manager
               return self.app(environ, start_response)
>>  return self.app(environ, start_response)
Module beaker.middleware:155 in __call__
<<                          headers.append(('Set-cookie', cookie))
                   return start_response(status, headers, exc_info)
               return self.wrap_app(environ, session_start_response)

           def _get_session(self):
>>  return self.wrap_app(environ, session_start_response)
Module routes.middleware:131 in __call__
<<                                               r'\1', oldpath)

               response = self.app(environ, start_response)

               # Wrapped in try as in rare cases the attribute will be gone already
>>  response = self.app(environ, start_response)
Module pylons.wsgiapp:125 in __call__
<<         
               controller = self.resolve(environ, start_response)
               response = self.dispatch(controller, environ, start_response)

               if 'paste.testing_variables' in environ and hasattr(response,
>>  response = self.dispatch(controller, environ, start_response)
Module pylons.wsgiapp:324 in dispatch
<<          if log_debug:
                   log.debug("Calling controller class with WSGI interface")
               return controller(environ, start_response)

           def load_test_env(self, environ):
>>  return controller(environ, start_response)
Module ckan.lib.base:346 in __call__
<<          try:
                   res = WSGIController.__call__(self, environ, start_response)
               finally:
                   model.Session.remove()
>>  res = WSGIController.__call__(self, environ, start_response)
Module pylons.controllers.core:221 in __call__
<<                  return response(environ, self.start_response)

               response = self._dispatch_call()
               if not start_response_called:
                   self.start_response = start_response
>>  response = self._dispatch_call()
Module pylons.controllers.core:172 in _dispatch_call
<<              req.environ['pylons.action_method'] = func

                   response = self._inspect_call(func)
               else:
                   if log_debug:
>>  response = self._inspect_call(func)
Module pylons.controllers.core:107 in _inspect_call
<<                        func.__name__, args)
               try:
                   result = self._perform_call(func, args)
               except HTTPException, httpe:
                   if log_debug:
>>  result = self._perform_call(func, args)
Module pylons.controllers.core:60 in _perform_call
<<          """Hide the traceback for everything above this method"""
               __traceback_hide__ = 'before_and_this'
               return func(**args)

           def _inspect_call(self, func):
>>  return func(**args)
Module ckanext.hdx_package.controllers.storage_controller:58 in file
<<          res = query.fetchone()
               if not res:
                   raise logic.NotFound

               # We need this as a resource object to check access so create a dummy obj and trick CKAN
>>  raise logic.NotFound
NotFound:
CGI Variables
CKAN_CURRENT_URL    '/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx'
CKAN_LANG   'en_AU'
CKAN_LANG_IS_DEFAULT    True
CONTENT_TYPE    '; charset=utf-8'
HTTP_ACCEPT     'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
HTTP_ACCEPT_ENCODING    'gzip, deflate'
HTTP_ACCEPT_LANGUAGE    'en-US,en;q=0.5'
HTTP_CACHE_CONTROL  'max-age=259200'
HTTP_CONNECTION     'close'
HTTP_COOKIE     '_ga=GA1.2.1843055497.1415374740'
HTTP_HOST   'data.hdx.rwlabs.org'
HTTP_SURROGATE_CAPABILITY   'rp-C1="Surrogate/1.0 ESI/1.0"'
HTTP_USER_AGENT     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:33.0) Gecko/20100101 Firefox/33.0'
HTTP_X_FORWARDED_FOR    '188.26.156.114'
PATH_INFO   '/storage/f/2014-10-31T21:31:36.370Z/health-care-workers-deaths.xlsx'
RAW_URI     '/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx'
REMOTE_ADDR     '127.0.0.1'
REMOTE_PORT     '53420'
REQUEST_METHOD  'GET'
SERVER_NAME     '0.0.0.0'
SERVER_PORT     '9221'
SERVER_PROTOCOL     'HTTP/1.0'
SERVER_SOFTWARE     'gunicorn/19.0.0'
WSGI Variables
application     <fanstatic.publisher.Delegator object at 0x56a9650>
beaker.cache    <beaker.cache.CacheManager object at 0x56a9b50>
beaker.get_session  <bound method SessionMiddleware._get_session of <beaker.middleware.SessionMiddleware object at 0x56a9a10>>
beaker.session  {'_accessed_time': 1415698969.863541, '_creation_time': 1415698969.863541}
fanstatic.needed    <fanstatic.core.NeededResources object at 0xde6d350>
gunicorn.socket     <socket._socketobject object at 0xbfb1ad0>
paste.cookies   (<SimpleCookie: _ga='GA1.2.1843055497.1415374740'>, '_ga=GA1.2.1843055497.1415374740')
paste.registry  <paste.registry.Registry object at 0xbc4fdd0>
paste.throw_errors  True
pylons.action_method    <bound method FileDownloadController.file of <ckanext.hdx_package.controllers.storage_controller.FileDownloadController object at 0xde6dfd0>>
pylons.controller   <ckanext.hdx_package.controllers.storage_controller.FileDownloadController object at 0xde6dfd0>
pylons.environ_config   {'session': 'beaker.session', 'cache': 'beaker.cache'}
pylons.pylons   <pylons.util.PylonsContext object at 0xde6d710>
pylons.routes_dict  {'action': u'file', 'controller': u'ckanext.hdx_package.controllers.storage_controller:FileDownloadController', 'label': u'2014-10-31T21:31:36.370Z/health-care-workers-deaths.xlsx'}
repoze.who.logger   <logging.Logger object at 0x54c3610>
repoze.who.plugins  {'openid': <OpenIdIdentificationPlugin 88879824>, 'friendlyform': <FriendlyFormPlugin 90853776>, 'ckan.lib.authenticator:UsernamePasswordAuthenticator': <ckan.lib.authenticator.UsernamePasswordAuthenticator object at 0x5832fd0>, 'auth_tkt': <AuthTktCookiePlugin 90853584>, 'ckan.lib.authenticator:OpenIDAuthenticator': <ckan.lib.authenticator.OpenIDAuthenticator object at 0x5832f90>}
routes.route    <routes.route.Route object at 0x543ded0>
routes.url  <routes.util.URLGenerator object at 0xde6d450>
webob._parsed_query_vars    (GET([]), '')
webob.adhoc_attrs   {'response': <Response at 0xde6d2d0 200 OK>, 'language': 'en-us'}
wsgi process    'Multiprocess'
wsgi.file_wrapper   <class 'gunicorn.http.wsgi.FileWrapper'>
wsgiorg.routing_args    (<routes.util.URLGenerator object at 0xde6d450>, {'action': u'file', 'controller': u'ckanext.hdx_package.controllers.storage_controller:FileDownloadController', 'label': u'2014-10-31T21:31:36.370Z/health-care-workers-deaths.xlsx'})
```



Discussion:
for 1. the problem with exception could be in storage_controller in method "file(self, label)"

@alexandru-m-g :
- Change GA trigger for downloads to handle direct URL download calls
- Change storage file naming to make links permanent to a given resource
- Delete outdated files


Resolving patch:
diff --git a/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py b/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py
--- a/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py
@@ -251,6 +251,34 @@ def new(self, data=None, errors=None, error_summary=None):
         # return render(self._new_template(package_type), extra_vars={'stage':
         # stage})
 
+    def _get_perma_link(self, dataset_id, resource_id):
+        perma_link = h.url_for(
+            'perma_storage_file', id=dataset_id, resource_id=resource_id)
+        if '://' not in perma_link:
+            perma_link = config.get(
+                'ckan.site_url', '') + perma_link
+        return perma_link
+
+    def _update_or_create_resource(self, context, data, dataset_id, resource_id):
+        if resource_id:
+            data['id'] = resource_id
+            # Added by HDX - adding perma_link
+            if 'resource_type' in data and u'file.upload' == data['resource_type']:
+                data['perma_link'] = self._get_perma_link(
+                    dataset_id, resource_id)
+
+            get_action('resource_update')(context, data)
+        else:
+            result_dict = get_action('resource_create')(context, data)
+
+            # Added by HDX - adding perma_link
+            # Now that we have a resource id we want to add the
+            # perma_link url to the resource
+            if 'resource_type' in result_dict and u'file.upload' == result_dict['resource_type']:
+                result_dict['perma_link'] = self._get_perma_link(
+                    dataset_id, result_dict['id'])
+                get_action('resource_update')(context, result_dict)
+
     def new_resource(self, id, data=None, errors=None, error_summary=None):
         ''' FIXME: This is a temporary action to allow styling of the
         forms. '''
@@ -305,11 +333,8 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
 
             data['package_id'] = id
             try:
-                if resource_id:
-                    data['id'] = resource_id
-                    get_action('resource_update')(context, data)
-                else:
-                    get_action('resource_create')(context, data)
+                self._update_or_create_resource(context, data, id, resource_id)
+
             except ValidationError, e:
                 errors = e.error_dict
                 error_summary = e.error_summary
@@ -444,11 +469,7 @@ def resource_edit(self, id, resource_id, data=None, errors=None,
 
             data['package_id'] = id
             try:
-                if resource_id:
-                    data['id'] = resource_id
-                    get_action('resource_update')(context, data)
-                else:
-                    get_action('resource_create')(context, data)
+                self._update_or_create_resource(context, data, id, resource_id)
             except ValidationError, e:
                 errors = e.error_dict
                 error_summary = e.error_summary
@@ -495,6 +516,14 @@ def resource_edit(self, id, resource_id, data=None, errors=None,
                 'error_summary': error_summary, 'action': 'new'}
         return render('package/resource_edit.html', extra_vars=vars)
 
+    def _create_perma_link_if_needed(self, dataset_id, resource):
+        if 'perma_link' not in resource and resource.get('resource_type', '') == 'file.upload':
+            domain = config.get('ckan.site_url', '')
+            if domain and domain in resource.get('url', ''):
+                perma_link = h.url_for(
+                    'perma_storage_file', id=dataset_id, resource_id=resource['id'])
+                resource['perma_link'] = perma_link
+
     def read(self, id, format='html'):
         if not format == 'html':
             ctype, extension, loader = \
@@ -547,16 +576,11 @@ def read(self, id, format='html'):
         c.current_package_id = c.pkg.id
         c.related_count = c.pkg.related_count
 
-        # can the resources be previewed?
         for resource in c.pkg_dict['resources']:
-            domain = config.get('ckan.site_url', '')
-            if domain and domain in resource.get('url', ''):
-                perma_link = h.url_for(
-                    'perma_storage_file', id=id, resource_id=resource['id'])
-                resource['perma_link'] = perma_link
-            else:
-                resource['perma_link'] = resource.get('url', '')
+            # create permalink if needed
+            self._create_perma_link_if_needed(id, resource)
 
+            # can the resources be previewed?
             resource['can_be_previewed'] = self._resource_preview(
                 {'resource': resource, 'package': c.pkg_dict})
         print c.pkg.related
diff --git a/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py b/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py
--- a/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py
@@ -77,7 +77,7 @@ def _download_file(self, res, label):
             logic.check_access('resource_show', context, data_dict)
         except logic.NotAuthorized:
             redirect_url = h.url_for(controller='user', action='login',
-                                     came_from=url)
+                                     came_from=resource.url)
             r = generate_response(303, u'Not authorized to read file ' + resource.id,
                                   other_headers={'Location': redirect_url, 'X-CKAN-Error': '403 Access Denied'})
             return r
@@ -128,7 +128,7 @@ def file(self, label):
             r = generate_response(404, u'File not found')
             return r
 
-        return self._download_file(label)
+        return self._download_file(res, label)
 
     def perma_file(self, id, resource_id):
         from sqlalchemy.engine import create_engine


Test patch:
diff --git a/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_private_resource.py b/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_private_resource.py
--- a/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_private_resource.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_private_resource.py
@@ -24,7 +24,6 @@ def _load_plugins(cls):
         hdx_test_base.load_plugin('hdx_package hdx_theme')
 
     def test_hdx_access_to_private_resource(self):
-        global package, resource, organization
 
         config['ofs.impl'] = 'pairtree'
         config['ofs.storage_dir'] = '/tmp'
@@ -41,7 +40,7 @@ def test_hdx_access_to_private_resource(self):
         dwld_url = h.url_for(controller='ckanext.hdx_package.controllers.storage_controller:FileDownloadController',
                              action='file', label='test_folder/hdx_test.csv')
 
-        # Testing access to private resource
+        # Testing access to private resource on old url
         try:
             self.app.get(
                 dwld_url,
@@ -61,6 +60,29 @@ def test_hdx_access_to_private_resource(self):
         except Exception, e:
             assert False
 
+        perma_link = h.url_for(controller='ckanext.hdx_package.controllers.storage_controller:FileDownloadController',
+                               action='perma_file', id='test_private_dataset_1', resource_id=resource['id'])
+
+        # Testing access to private resource on perma_link
+        try:
+            self.app.get(
+                perma_link,
+                extra_environ={'Authorization': str(testsysadmin.apikey)})
+        except Exception, e:
+            # The file doesn't really exist
+            assert '404' in e.args[0], 'File not found'
+        try:
+            result = self.app.get(
+                perma_link, extra_environ={'Authorization': str(tester.apikey)})
+            assert '403 Access Denied' in str(result)
+        except Exception, e:
+            assert False
+        try:
+            result = self.app.get(perma_link)
+            assert '403 Access Denied' in str(result)
+        except Exception, e:
+            assert False
+
         # Testing access to the private resource API
         tests.call_action_api(self.app, 'resource_show', id=resource['id'],
                               status=403)
diff --git a/ckanext-hdx_package/ckanext/hdx_package/tests/test_pages/test_perma_link_display.py b/ckanext-hdx_package/ckanext/hdx_package/tests/test_pages/test_perma_link_display.py
new file mode 100644
--- /dev/null
+++ b/ckanext-hdx_package/ckanext/hdx_package/tests/test_pages/test_perma_link_display.py
@@ -0,0 +1,41 @@
+'''
+Created on Dec 2, 2014
+
+@author: alexandru-m-g
+'''
+
+import logging as logging
+
+import ckan.model as model
+import ckan.lib.helpers as h
+
+import ckanext.hdx_theme.tests.hdx_test_base as hdx_test_base
+import ckanext.hdx_theme.tests.hdx_test_with_inds_and_orgs as hdx_test_with_inds_and_orgs
+
+log = logging.getLogger(__name__)
+
+
+class TestPermaLinkDisplay(hdx_test_with_inds_and_orgs.HDXWithIndsAndOrgsTest):
+
+    @classmethod
+    def _load_plugins(cls):
+        hdx_test_base.load_plugin('hdx_package hdx_theme')
+
+    def test_hdx_access_to_private_resource(self):
+        testsysadmin = model.User.by_name('testsysadmin')
+        context = {
+            'model': model, 'session': model.Session, 'user': 'testsysadmin'}
+        resource = self._get_action('resource_show')(
+            context, {'id': 'hdx_test.csv'})
+        perma_link = h.url_for(controller='ckanext.hdx_package.controllers.storage_controller:FileDownloadController',
+                               action='perma_file', id='test_private_dataset_1', resource_id=resource['id'])
+
+        test_dataset_url = h.url_for(controller='ckanext.hdx_package.controllers.dataset_controller:DatasetController',
+                                     action='read', id='test_private_dataset_1')
+
+        try:
+            result = self.app.get(
+                test_dataset_url, extra_environ={'Authorization': str(testsysadmin.apikey)})
+            assert perma_link in str(result)
+        except Exception, e:
+            assert False, str(e)