Repository: OCHA-DAP/hdx-ckan

Issue:
Server Logs: google bot accessing a resource
Below is the full error we are getting in logs. It might have two parts:
1. An exception that must be catched and nicer displayed
2. we must inform google that the resource has another url. Do we want nicer url?
For example the url google is trying to access: http://data.hdx.rwlabs.org/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx
is changed to this one:
http://data.hdx.rwlabs.org/storage/f/2014-11-07T18%3A43%3A40.574Z/health-care-workers-deaths.xlsx

```
URL: http://data.hdx.rwlabs.org/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx
Module weberror.errormiddleware:162 in __call__
<<              __traceback_supplement__ = Supplement, self, environ
                   sr_checker = ResponseStartChecker(start_response)
                   app_iter = self.application(environ, sr_checker)
                   return self.make_catching_iter(app_iter, environ, sr_checker)
               except:
>>  app_iter = self.application(environ, sr_checker)
Module webob.dec:147 in __call__
<<                  if self.middleware_wraps:
                           args = (self.middleware_wraps,) + args
                       resp = self.call_func(req, *args, **self.kwargs)
                   except webob.exc.HTTPException, resp:
                       pass
>>  resp = self.call_func(req, *args, **self.kwargs)
Module webob.dec:208 in call_func
<<          """Call the wrapped function; override this in a subclass to
               change how the function is called."""
               return self.func(req, *args, **kwargs)

           def clone(self, func=None, **kw):
>>  return self.func(req, *args, **kwargs)
Module fanstatic.publisher:234 in __call__
<<              # the trigger segment is not in the URL, so we delegate
                   # to the original application
                   return request.get_response(self.app)
               # the trigger is in there, so let whatever is behind the
               # trigger be handled by the publisher
>>  return request.get_response(self.app)
Module webob.request:1053 in get_response
<<          else:
                   status, headers, app_iter = self.call_application(
                       application, catch_exc_info=False)
               return self.ResponseClass(
                   status=status, headerlist=list(headers), app_iter=app_iter,
>>  application, catch_exc_info=False)
Module webob.request:1022 in call_application
<<              captured[:] = [status, headers, exc_info]
                   return output.append
               app_iter = application(self.environ, start_response)
               if output or not captured:
                   try:
>>  app_iter = application(self.environ, start_response)
Module webob.dec:147 in __call__
<<                  if self.middleware_wraps:
                           args = (self.middleware_wraps,) + args
                       resp = self.call_func(req, *args, **self.kwargs)
                   except webob.exc.HTTPException, resp:
                       pass
>>  resp = self.call_func(req, *args, **self.kwargs)
Module webob.dec:208 in call_func
<<          """Call the wrapped function; override this in a subclass to
               change how the function is called."""
               return self.func(req, *args, **kwargs)

           def clone(self, func=None, **kw):
>>  return self.func(req, *args, **kwargs)
Module fanstatic.injector:54 in __call__
<<          # Get the response from the wrapped application:
               response = request.get_response(self.app)

               # We only continue if the content-type is appropriate.
>>  response = request.get_response(self.app)
Module webob.request:1053 in get_response
<<          else:
                   status, headers, app_iter = self.call_application(
                       application, catch_exc_info=False)
               return self.ResponseClass(
                   status=status, headerlist=list(headers), app_iter=app_iter,
>>  application, catch_exc_info=False)
Module webob.request:1022 in call_application
<<              captured[:] = [status, headers, exc_info]
                   return output.append
               app_iter = application(self.environ, start_response)
               if output or not captured:
                   try:
>>  app_iter = application(self.environ, start_response)
Module beaker.middleware:73 in __call__
<<                                                     self.cache_manager)
               environ[self.environ_key] = self.cache_manager
               return self.app(environ, start_response)
>>  return self.app(environ, start_response)
Module beaker.middleware:155 in __call__
<<                          headers.append(('Set-cookie', cookie))
                   return start_response(status, headers, exc_info)
               return self.wrap_app(environ, session_start_response)

           def _get_session(self):
>>  return self.wrap_app(environ, session_start_response)
Module routes.middleware:131 in __call__
<<                                               r'\1', oldpath)

               response = self.app(environ, start_response)

               # Wrapped in try as in rare cases the attribute will be gone already
>>  response = self.app(environ, start_response)
Module pylons.wsgiapp:125 in __call__
<<         
               controller = self.resolve(environ, start_response)
               response = self.dispatch(controller, environ, start_response)

               if 'paste.testing_variables' in environ and hasattr(response,
>>  response = self.dispatch(controller, environ, start_response)
Module pylons.wsgiapp:324 in dispatch
<<          if log_debug:
                   log.debug("Calling controller class with WSGI interface")
               return controller(environ, start_response)

           def load_test_env(self, environ):
>>  return controller(environ, start_response)
Module ckan.lib.base:346 in __call__
<<          try:
                   res = WSGIController.__call__(self, environ, start_response)
               finally:
                   model.Session.remove()
>>  res = WSGIController.__call__(self, environ, start_response)
Module pylons.controllers.core:221 in __call__
<<                  return response(environ, self.start_response)

               response = self._dispatch_call()
               if not start_response_called:
                   self.start_response = start_response
>>  response = self._dispatch_call()
Module pylons.controllers.core:172 in _dispatch_call
<<              req.environ['pylons.action_method'] = func

                   response = self._inspect_call(func)
               else:
                   if log_debug:
>>  response = self._inspect_call(func)
Module pylons.controllers.core:107 in _inspect_call
<<                        func.__name__, args)
               try:
                   result = self._perform_call(func, args)
               except HTTPException, httpe:
                   if log_debug:
>>  result = self._perform_call(func, args)
Module pylons.controllers.core:60 in _perform_call
<<          """Hide the traceback for everything above this method"""
               __traceback_hide__ = 'before_and_this'
               return func(**args)

           def _inspect_call(self, func):
>>  return func(**args)
Module ckanext.hdx_package.controllers.storage_controller:58 in file
<<          res = query.fetchone()
               if not res:
                   raise logic.NotFound

               # We need this as a resource object to check access so create a dummy obj and trick CKAN
>>  raise logic.NotFound
NotFound:
CGI Variables
CKAN_CURRENT_URL    '/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx'
CKAN_LANG   'en_AU'
CKAN_LANG_IS_DEFAULT    True
CONTENT_TYPE    '; charset=utf-8'
HTTP_ACCEPT     'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
HTTP_ACCEPT_ENCODING    'gzip, deflate'
HTTP_ACCEPT_LANGUAGE    'en-US,en;q=0.5'
HTTP_CACHE_CONTROL  'max-age=259200'
HTTP_CONNECTION     'close'
HTTP_COOKIE     '_ga=GA1.2.1843055497.1415374740'
HTTP_HOST   'data.hdx.rwlabs.org'
HTTP_SURROGATE_CAPABILITY   'rp-C1="Surrogate/1.0 ESI/1.0"'
HTTP_USER_AGENT     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:33.0) Gecko/20100101 Firefox/33.0'
HTTP_X_FORWARDED_FOR    '188.26.156.114'
PATH_INFO   '/storage/f/2014-10-31T21:31:36.370Z/health-care-workers-deaths.xlsx'
RAW_URI     '/storage/f/2014-10-31T21%3A31%3A36.370Z/health-care-workers-deaths.xlsx'
REMOTE_ADDR     '127.0.0.1'
REMOTE_PORT     '53420'
REQUEST_METHOD  'GET'
SERVER_NAME     '0.0.0.0'
SERVER_PORT     '9221'
SERVER_PROTOCOL     'HTTP/1.0'
SERVER_SOFTWARE     'gunicorn/19.0.0'
WSGI Variables
application     <fanstatic.publisher.Delegator object at 0x56a9650>
beaker.cache    <beaker.cache.CacheManager object at 0x56a9b50>
beaker.get_session  <bound method SessionMiddleware._get_session of <beaker.middleware.SessionMiddleware object at 0x56a9a10>>
beaker.session  {'_accessed_time': 1415698969.863541, '_creation_time': 1415698969.863541}
fanstatic.needed    <fanstatic.core.NeededResources object at 0xde6d350>
gunicorn.socket     <socket._socketobject object at 0xbfb1ad0>
paste.cookies   (<SimpleCookie: _ga='GA1.2.1843055497.1415374740'>, '_ga=GA1.2.1843055497.1415374740')
paste.registry  <paste.registry.Registry object at 0xbc4fdd0>
paste.throw_errors  True
pylons.action_method    <bound method FileDownloadController.file of <ckanext.hdx_package.controllers.storage_controller.FileDownloadController object at 0xde6dfd0>>
pylons.controller   <ckanext.hdx_package.controllers.storage_controller.FileDownloadController object at 0xde6dfd0>
pylons.environ_config   {'session': 'beaker.session', 'cache': 'beaker.cache'}
pylons.pylons   <pylons.util.PylonsContext object at 0xde6d710>
pylons.routes_dict  {'action': u'file', 'controller': u'ckanext.hdx_package.controllers.storage_controller:FileDownloadController', 'label': u'2014-10-31T21:31:36.370Z/health-care-workers-deaths.xlsx'}
repoze.who.logger   <logging.Logger object at 0x54c3610>
repoze.who.plugins  {'openid': <OpenIdIdentificationPlugin 88879824>, 'friendlyform': <FriendlyFormPlugin 90853776>, 'ckan.lib.authenticator:UsernamePasswordAuthenticator': <ckan.lib.authenticator.UsernamePasswordAuthenticator object at 0x5832fd0>, 'auth_tkt': <AuthTktCookiePlugin 90853584>, 'ckan.lib.authenticator:OpenIDAuthenticator': <ckan.lib.authenticator.OpenIDAuthenticator object at 0x5832f90>}
routes.route    <routes.route.Route object at 0x543ded0>
routes.url  <routes.util.URLGenerator object at 0xde6d450>
webob._parsed_query_vars    (GET([]), '')
webob.adhoc_attrs   {'response': <Response at 0xde6d2d0 200 OK>, 'language': 'en-us'}
wsgi process    'Multiprocess'
wsgi.file_wrapper   <class 'gunicorn.http.wsgi.FileWrapper'>
wsgiorg.routing_args    (<routes.util.URLGenerator object at 0xde6d450>, {'action': u'file', 'controller': u'ckanext.hdx_package.controllers.storage_controller:FileDownloadController', 'label': u'2014-10-31T21:31:36.370Z/health-care-workers-deaths.xlsx'})
```



Discussion:
for 1. the problem with exception could be in storage_controller in method "file(self, label)"

@alexandru-m-g :
- Change GA trigger for downloads to handle direct URL download calls
- Change storage file naming to make links permanent to a given resource
- Delete outdated files


Resolving patch:
diff --git a/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py b/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py
--- a/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py
+++ b/ckanext-hdx_package/ckanext/hdx_package/controllers/storage_controller.py
@@ -77,7 +77,7 @@ def _download_file(self, res, label):
             logic.check_access('resource_show', context, data_dict)
         except logic.NotAuthorized:
             redirect_url = h.url_for(controller='user', action='login',
-                                     came_from=url)
+                                     came_from=resource.url)
             r = generate_response(303, u'Not authorized to read file ' + resource.id,
                                   other_headers={'Location': redirect_url, 'X-CKAN-Error': '403 Access Denied'})
             return r
@@ -128,7 +128,7 @@ def file(self, label):
             r = generate_response(404, u'File not found')
             return r
 
-        return self._download_file(label)
+        return self._download_file(res, label)
 
     def perma_file(self, id, resource_id):
         from sqlalchemy.engine import create_engine


Test patch: