Repository: simphony/simphony-remote

Issue:
Formalise error handling with the logger and the refnumber for the issue
Currently when an error takes place in a number of places we generate a `refnumber` that can be displayed in the user when an error takes place. This `refnumber` is at the same time logged next to the related error. This will help to pinpoint the place in the logs where a specific error (reported by a user) took place and simplify debugging.

To achieve this I would suggest one of the following:

- Wrap the python `Logger`  logging functions so that they generate a refnumber, pass it to the logger and then return it to the caller.
- Provide a generic `log(self, level, msg, *args, **kwargs)` function in the `BaseHandler` that will generate the refnumber pass it to a python logger and return it.

I think that the second implementation is simpler and easier to test. 



Comments:

Patch:
diff --git a/remoteappmanager/application.py b/remoteappmanager/application.py
index 7666a2a..abfcecf 100644
--- a/remoteappmanager/application.py
+++ b/remoteappmanager/application.py
@@ -19,6 +19,7 @@ from remoteappmanager.services.reverse_proxy import ReverseProxy
 from remoteappmanager import rest
 from remoteappmanager.rest import registry
 from remoteappmanager import restresources
+from remoteappmanager.utils import url_path_join
 
 
 class Application(web.Application, LoggingMixin):
@@ -103,7 +104,6 @@ class Application(web.Application, LoggingMixin):
         return ReverseProxy(
             endpoint_url=self.command_line_config.proxy_api_url,
             auth_token=auth_token,
-            base_urlpath=self.command_line_config.base_urlpath
         )
 
     @default("hub")
@@ -149,6 +149,16 @@ class Application(web.Application, LoggingMixin):
 
         tornado.ioloop.IOLoop.current().start()
 
+    def urlpath_for_object(self, object):
+        """
+        Resolves the absolute url path of a given object.
+        The object must have a urlpath property.
+        """
+
+        return url_path_join(
+            self.command_line_config.base_urlpath,
+            object.urlpath)
+
     # Private
     def _get_handlers(self):
         """Returns the registered handlers"""
diff --git a/remoteappmanager/handlers/home_handler.py b/remoteappmanager/handlers/home_handler.py
index c1648de..c21824b 100644
--- a/remoteappmanager/handlers/home_handler.py
+++ b/remoteappmanager/handlers/home_handler.py
@@ -1,6 +1,5 @@
 import socket
 import os
-import uuid
 from datetime import timedelta
 
 import errno
@@ -45,12 +44,11 @@ class HomeHandler(BaseHandler):
 
         try:
             yield handler(options)
-        except Exception as e:
+        except Exception as exc:
             # Create a random reference number for support
-            ref = str(uuid.uuid1())
-            self.log.exception("Failed with POST action: {0}. {1} "
-                               "Ref: {2}".format(
-                                   action, str(e), ref))
+            ref = self.log.issue(
+                "Failed with POST action {}".format(action),
+                exc)
 
             images_info = yield self._get_images_info()
 
@@ -63,7 +61,7 @@ class HomeHandler(BaseHandler):
                         images_info=images_info,
                         error_message=message.format(
                             action=action,
-                            error_type=type(e).__name__,
+                            error_type=type(exc).__name__,
                             ref=ref))
 
     # Subhandling after post
@@ -112,7 +110,11 @@ class HomeHandler(BaseHandler):
 
         # The server is up and running. Now contact the proxy and add
         # the container url to it.
-        urlpath = yield self.application.reverse_proxy.add_container(container)
+        urlpath = url_path_join(
+            self.application.command_line_config.base_urlpath,
+            container.urlpath)
+        yield self.application.reverse_proxy.register(
+            urlpath, container.host_url)
 
         # Redirect the user
         self.log.info('Redirecting to {}'.format(urlpath))
@@ -136,7 +138,11 @@ class HomeHandler(BaseHandler):
         yield self._wait_for_container_ready(container)
 
         # in case the reverse proxy is not already set up
-        urlpath = yield self.application.reverse_proxy.add_container(container)
+        urlpath = url_path_join(
+            self.application.command_line_config.base_urlpath,
+            container.urlpath)
+        yield self.application.reverse_proxy.register(
+            urlpath, container.host_url)
 
         self.log.info('Redirecting to {}'.format(urlpath))
         self.redirect(urlpath)
@@ -155,13 +161,10 @@ class HomeHandler(BaseHandler):
             self.finish("Unable to view the application")
             return
 
-        try:
-            yield app.reverse_proxy.remove_container(container)
-        except HTTPError as http_error:
-            # The reverse proxy may be absent to start with
-            if http_error.code != 404:
-                raise http_error
-
+        urlpath = url_path_join(
+            self.application.command_line_config.base_urlpath,
+            container.urlpath)
+        yield app.reverse_proxy.unregister(urlpath)
         yield container_manager.stop_and_remove_container(container.docker_id)
 
         # We don't have fancy stuff at the moment to change the button, so
diff --git a/remoteappmanager/logging/logging_mixin.py b/remoteappmanager/logging/logging_mixin.py
index 6339a43..fddc6f6 100644
--- a/remoteappmanager/logging/logging_mixin.py
+++ b/remoteappmanager/logging/logging_mixin.py
@@ -1,3 +1,6 @@
+import uuid
+import types
+
 from traitlets import (
     HasTraits,
     Instance,
@@ -5,6 +8,25 @@ from traitlets import (
 )
 
 
+def issue(self, message, exc=None):
+    """Accepts a message that will be logged with an additional reference
+    code for easy log lookup.
+
+    The identifier will be returned for inclusion in user-visible
+    error messages.
+    """
+    ref = str(uuid.uuid1())
+
+    if exc is None:
+        err_message = "{}. Ref: {}".format(message, ref)
+    else:
+        err_message = "{} : {}. Ref: {}".format(
+            message, str(exc), ref)
+
+    self.error(err_message)
+    return ref
+
+
 class LoggingMixin(HasTraits):
     """A HasTrait class that provides logging. Used as a mixin.
     """
@@ -13,5 +35,9 @@ class LoggingMixin(HasTraits):
 
     @default('log')
     def _log_default(self):
-        from traitlets import log
-        return log.get_logger()
+        from tornado.log import app_log
+
+        # monkey patch the logger to provide an additional method that handles
+        # issues
+        app_log.issue = types.MethodType(issue, app_log)
+        return app_log
diff --git a/remoteappmanager/restresources/container.py b/remoteappmanager/restresources/container.py
index 3191c6b..cbf224a 100644
--- a/remoteappmanager/restresources/container.py
+++ b/remoteappmanager/restresources/container.py
@@ -36,7 +36,11 @@ class Container(Resource):
                                                 policy,
                                                 mapping_id)
         yield self._wait_for_container_ready(container)
-        yield self.application.reverse_proxy.add_container(container)
+        urlpath = url_path_join(
+            self.application.command_line_config.base_urlpath,
+            container.urlpath)
+        yield self.application.reverse_proxy.register(urlpath,
+                                                      container.host_url)
 
         return container.url_id
 
@@ -56,13 +60,15 @@ class Container(Resource):
     @gen.coroutine
     def delete(self, identifier):
         """Stop the container."""
-        app = self.application
         container = yield self._container_from_url_id(identifier)
         if not container:
             raise exceptions.NotFound()
 
-        yield app.reverse_proxy.remove_container(container)
-        yield app.container_manager.stop_and_remove_container(
+        urlpath = url_path_join(
+            self.application.command_line_config.base_urlpath,
+            container.urlpath)
+        yield self.application.reverse_proxy.unregister(urlpath)
+        yield self.application.container_manager.stop_and_remove_container(
             container.docker_id)
 
     @gen.coroutine
diff --git a/remoteappmanager/services/reverse_proxy.py b/remoteappmanager/services/reverse_proxy.py
index 64807c3..4bff151 100644
--- a/remoteappmanager/services/reverse_proxy.py
+++ b/remoteappmanager/services/reverse_proxy.py
@@ -5,7 +5,6 @@ from jupyterhub import orm as jupyterhub_orm
 from traitlets import HasTraits, Unicode
 
 from remoteappmanager.logging.logging_mixin import LoggingMixin
-from remoteappmanager.utils import url_path_join
 
 
 class ReverseProxy(LoggingMixin, HasTraits):
@@ -17,9 +16,6 @@ class ReverseProxy(LoggingMixin, HasTraits):
     #: The authorization token to authenticate the request
     auth_token = Unicode()
 
-    #: The prefix for the url added to the passed object relative .url()
-    base_urlpath = Unicode('/')
-
     def __init__(self, *args, **kwargs):
         """Initializes the reverse proxy connection object."""
         super().__init__(*args, **kwargs)
@@ -31,73 +27,59 @@ class ReverseProxy(LoggingMixin, HasTraits):
             api_server=_server_from_url(self.endpoint_url)
         )
 
-        self.log.info("Reverse proxy setup on {} with base url {}".format(
+        self.log.info("Reverse proxy setup on {}".format(
             self.endpoint_url,
-            self.base_urlpath
         ))
 
     @gen.coroutine
-    def remove_container(self, container):
-        """Removes a container from the reverse proxy at the associated url.
+    def register(self, urlpath, target_host_url):
+        """Register a given urlpath to redirect to a different target host.
+        The operation is idempotent.
 
         Parameters
         ----------
-        container : remoteappmanager.docker.container.Container
-            A container object.
+        urlpath: str
+            The absolute path of the url (e.g. /my/internal/service/)"
+
+        target_host_url:
+            The host to redirect to, e.g. http://127.0.0.1:31233/service/
         """
-        proxy = self._reverse_proxy
+        self.log.info("Registering {} redirection to {}".format(
+            urlpath,
+            target_host_url))
 
-        urlpath = url_path_join(self.base_urlpath, container.urlpath)
-        self.log.info("Unregistering url {} to {} on reverse proxy.".format(
+        yield self._reverse_proxy.api_request(
             urlpath,
-            container.host_url
-        ))
+            method='POST',
+            body=dict(
+                target=target_host_url,
+            )
+        )
+
+    @gen.coroutine
+    def unregister(self, urlpath):
+        """Unregisters a previously registered urlpath.
+        If the urlpath is not found in the reverse proxy, it will not raise
+        an error, but it will log the unexpected circumstance.
+
+        Parameters
+        ----------
+        urlpath: str
+            The absolute path of the url (e.g. /my/internal/service/"
+        """
+        self.log.info("Deregistering {} redirection".format(urlpath))
 
         try:
-            yield proxy.api_request(urlpath, method='DELETE')
+            yield self._reverse_proxy.api_request(urlpath, method='DELETE')
         except httpclient.HTTPError as e:
             if e.code == 404:
                 self.log.warning("Could not find urlpath {} when removing"
                                  " container. In any case, the reverse proxy"
                                  " does not map the url. Continuing".format(
-                                    urlpath))
+                                     urlpath))
             else:
                 raise e
 
-    @gen.coroutine
-    def add_container(self, container):
-        """Adds the url associated to a given container on the reverse proxy.
-
-        Parameters
-        ----------
-        container : remoteappmanager.docker.container.Container
-            A container object.
-
-        Returns
-        -------
-        urlpath : str
-            The absolute url path of the container as registered on the reverse
-            proxy.
-        """
-
-        proxy = self._reverse_proxy
-        urlpath = url_path_join(self.base_urlpath, container.urlpath)
-
-        self.log.info("Registering url {} to {} on reverse proxy.".format(
-            urlpath,
-            container.host_url
-        ))
-
-        yield proxy.api_request(
-            urlpath,
-            method='POST',
-            body=dict(
-                target=container.host_url,
-            )
-        )
-
-        return urlpath
-
 
 def _server_from_url(url):
     """Creates a orm.Server from a given url"""


Test patch:
diff --git a/tests/handlers/test_home_handler.py b/tests/handlers/test_home_handler.py
index 95807b8..596b8db 100644
--- a/tests/handlers/test_home_handler.py
+++ b/tests/handlers/test_home_handler.py
@@ -7,10 +7,10 @@ from remoteappmanager.docker.container import Container
 from remoteappmanager.docker.container_manager import ContainerManager
 from remoteappmanager.docker.image import Image
 from remoteappmanager.services.hub import Hub
-from remoteappmanager.services.reverse_proxy import ReverseProxy
 
 from remoteappmanager.application import Application
 from tests import utils
+from tests.mock.mock_reverse_proxy import MockReverseProxy
 from tests.temp_mixin import TempMixin
 
 
@@ -38,9 +38,7 @@ class TestHomeHandler(TempMixin, utils.AsyncHTTPTestCase):
         file_config.accounting_kwargs = {'url': "sqlite:///"+sqlite_file_path}
 
         app = Application(command_line_config, file_config)
-        app.reverse_proxy = mock.Mock(spec=ReverseProxy)
-        app.reverse_proxy.add_container = utils.mock_coro_factory("/")
-        app.reverse_proxy.remove_container = utils.mock_coro_factory()
+        app.reverse_proxy = MockReverseProxy()
         app.hub = mock.Mock(spec=Hub)
         app.hub.verify_token = utils.mock_coro_factory({
             'pending': None,
@@ -107,7 +105,7 @@ class TestHomeHandler(TempMixin, utils.AsyncHTTPTestCase):
                        ".HomeHandler"
                        ".redirect") as redirect:
 
-            self.assertFalse(self._app.reverse_proxy.add_container.called)
+            self.assertFalse(self._app.reverse_proxy.register.called)
             self.fetch("/user/username/",
                        method="POST",
                        headers={
@@ -115,7 +113,7 @@ class TestHomeHandler(TempMixin, utils.AsyncHTTPTestCase):
                        },
                        body=body)
 
-            self.assertTrue(self._app.reverse_proxy.add_container.called)
+            self.assertTrue(self._app.reverse_proxy.register.called)
             self.assertTrue(redirect.called)
 
     def test_post_failed_auth(self):
@@ -156,7 +154,7 @@ class TestHomeHandler(TempMixin, utils.AsyncHTTPTestCase):
                        },
                        body=body)
 
-            self.assertTrue(self._app.reverse_proxy.remove_container.called)
+            self.assertTrue(self._app.reverse_proxy.unregister.called)
             self.assertTrue(redirect.called)
 
     def test_post_view(self):
@@ -184,5 +182,5 @@ class TestHomeHandler(TempMixin, utils.AsyncHTTPTestCase):
                        },
                        body=body)
 
-            self.assertTrue(self._app.reverse_proxy.add_container.called)
+            self.assertTrue(self._app.reverse_proxy.register.called)
             self.assertTrue(redirect.called)
diff --git a/tests/logging/__init__.py b/tests/logging/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/logging/test_logging_mixin.py b/tests/logging/test_logging_mixin.py
new file mode 100644
index 0000000..833562e
--- /dev/null
+++ b/tests/logging/test_logging_mixin.py
@@ -0,0 +1,19 @@
+import unittest
+from unittest import mock
+
+from remoteappmanager.logging.logging_mixin import LoggingMixin
+
+
+class Logged(LoggingMixin):
+    pass
+
+
+class TestLoggingMixin(unittest.TestCase):
+    def test_issue(self):
+        l = Logged()
+        with mock.patch("tornado.log.app_log.error") as mock_error:
+            ref = l.log.issue("hello")
+            self.assertIn("hello. Ref: "+str(ref), mock_error.call_args[0][0])
+            ref = l.log.issue("hello", Exception("Boom!"))
+            self.assertIn("hello : Boom!. Ref: "+str(ref),
+                          mock_error.call_args[0][0])
diff --git a/tests/mock/__init__.py b/tests/mock/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/mock/mock_reverse_proxy.py b/tests/mock/mock_reverse_proxy.py
new file mode 100644
index 0000000..ee624a6
--- /dev/null
+++ b/tests/mock/mock_reverse_proxy.py
@@ -0,0 +1,15 @@
+from unittest import mock
+
+from remoteappmanager.services.reverse_proxy import ReverseProxy
+from tests import utils
+
+
+def MockReverseProxy():
+    """Constructor. Returns a mock reverse proxy implementation.
+    Named as a class for potential future expansion in mock implementation."""
+
+    mock_revproxy = mock.Mock(spec=ReverseProxy)
+    mock_revproxy.register = utils.mock_coro_factory("/")
+    mock_revproxy.unregister = utils.mock_coro_factory()
+
+    return mock_revproxy
diff --git a/tests/restmodel/test_container.py b/tests/restmodel/test_container.py
index ba87141..7ae9a96 100644
--- a/tests/restmodel/test_container.py
+++ b/tests/restmodel/test_container.py
@@ -2,6 +2,7 @@ from unittest.mock import Mock, patch
 
 from remoteappmanager.restresources import Container
 from tests import utils
+from tests.mock.mock_reverse_proxy import MockReverseProxy
 from tornado import web, escape
 
 from remoteappmanager import rest
@@ -28,11 +29,10 @@ class TestContainer(AsyncHTTPTestCase):
         app = web.Application(handlers=handlers)
         app.file_config = Mock()
         app.file_config.network_timeout = 5
+        app.urlpath_for_object = Mock(return_value="/urlpath_for_object/")
         app.command_line_config = Mock()
         app.command_line_config.base_urlpath = "/"
-        app.reverse_proxy = Mock()
-        app.reverse_proxy.add_container = mock_coro_factory()
-        app.reverse_proxy.remove_container = mock_coro_factory()
+        app.reverse_proxy = MockReverseProxy()
         container = Mock()
         container.urlpath = "containers/12345"
         container.url_id = "12345"
diff --git a/tests/services/test_reverse_proxy.py b/tests/services/test_reverse_proxy.py
index 39ff093..71b0298 100644
--- a/tests/services/test_reverse_proxy.py
+++ b/tests/services/test_reverse_proxy.py
@@ -4,8 +4,6 @@ from jupyterhub import orm
 from remoteappmanager.services.reverse_proxy import ReverseProxy
 from tornado import gen, testing
 
-from remoteappmanager.docker.container import Container
-
 
 class TestReverseProxy(testing.AsyncTestCase):
     @testing.gen_test
@@ -22,11 +20,11 @@ class TestReverseProxy(testing.AsyncTestCase):
         reverse_proxy._reverse_proxy = Mock(spec=orm.Proxy)
         reverse_proxy._reverse_proxy.api_request = mock_api_request
 
-        container = Container(docker_id="12345", base_urlpath="/foo/")
-        yield reverse_proxy.add_container(container)
+        yield reverse_proxy.register("/hello/from/me/",
+                                     "http://localhost:12312/")
 
         self.assertEqual(coroutine_out["kwargs"]["method"], "POST")
 
-        yield reverse_proxy.remove_container(container)
+        yield reverse_proxy.unregister("/hello/from/me/")
 
         self.assertEqual(coroutine_out["kwargs"]["method"], "DELETE")