Repository: learningequality/kolibri

Issue:
Feature request: make LISTEN_ADDRESS an option
At the moment, Kolibri's server always listens on all network interfaces, with the following code in [utils/server.py](https://github.com/learningequality/kolibri/blob/release-v0.14.x/kolibri/utils/server.py#L64-L65):

```
# Currently non-configurable until we know how to properly handle this
LISTEN_ADDRESS = "0.0.0.0"
```

On Windows, listening on all interfaces requires a firewall exception, meaning a user is greeted with a scary firewall dialog when they first run Kolibri. This makes sense usually (we want Kolibri instances to talk to each other and firewall dialogs are not exactly uncommon in Windows), but it is problematic in an application where we are providing pre-installed Kolibri instances for _offline_ users who don't need that functionality. So, it would be nice if the listen address was configurable :)



Discussion:
@jamalex pointed out that having this be configurable as a list of interfaces (as opposed to just a single value) would also be helpful.

It may be that we would need to run separate WSGI servers to do this more specific binding (which would then also open the possibility for the port and zip_port options to be lists as well).
Closed in error

Resolving patch:
diff --git a/kolibri/utils/cli.py b/kolibri/utils/cli.py
--- a/kolibri/utils/cli.py
+++ b/kolibri/utils/cli.py
@@ -255,12 +255,15 @@ def start(port, zip_port, background):
     zip_port = (
         OPTIONS["Deployment"]["ZIP_CONTENT_PORT"] if zip_port is None else zip_port
     )
-    server.start(
-        port=port,
-        zip_port=zip_port,
-        serve_http=OPTIONS["Server"]["CHERRYPY_START"],
-        background=background,
-    )
+    try:
+        server.start(
+            port=port,
+            zip_port=zip_port,
+            serve_http=OPTIONS["Server"]["CHERRYPY_START"],
+            background=background,
+        )
+    except server.PortOccupied:
+        sys.exit(1)
 
 
 @main.command(cls=KolibriCommand, help="Stop the Kolibri process")
@@ -340,8 +343,10 @@ def services(port, background):
     port = OPTIONS["Deployment"]["HTTP_PORT"] if port is None else port
 
     logger.info("Starting Kolibri background services")
-
-    server.start(port=port, zip_port=0, serve_http=False, background=background)
+    try:
+        server.start(port=port, zip_port=0, serve_http=False, background=background)
+    except server.PortOccupied:
+        sys.exit(1)
 
 
 @main.command(cls=KolibriCommand, help="Restart the Kolibri process")
diff --git a/kolibri/utils/options.py b/kolibri/utils/options.py
--- a/kolibri/utils/options.py
+++ b/kolibri/utils/options.py
@@ -540,6 +540,14 @@ def multiprocess_bool(value):
                 Value can either be a number suffixed with a unit (e.g. MB, GB, TB) or an integer number of bytes.
             """,
         },
+        "LISTEN_ADDRESS": {
+            "type": "ip_addr",
+            "default": "0.0.0.0",
+            "description": """
+                The address that the server should listen on. This can be used to restrict access to the server
+                to a specific network interface.
+            """,
+        },
     },
     "Python": {
         "PICKLE_PROTOCOL": {
diff --git a/kolibri/utils/server.py b/kolibri/utils/server.py
--- a/kolibri/utils/server.py
+++ b/kolibri/utils/server.py
@@ -68,9 +68,6 @@
 # might not have made it to the log file!
 DAEMON_LOG = os.path.join(conf.LOG_ROOT, "daemon.txt")
 
-# Currently non-configurable until we know how to properly handle this
-LISTEN_ADDRESS = "0.0.0.0"
-
 status_messages = {
     STATUS_RUNNING: "OK, running",
     STATUS_STOPPED: "Stopped",
@@ -102,6 +99,18 @@ def __init__(self, status_code):
         super(NotRunning, self).__init__()
 
 
+class PortOccupied(OSError):
+    pass
+
+
+class RunningException(PortOccupied):
+    """
+    Raised when server already appears to be running
+    """
+
+    pass
+
+
 class Server(BaseServer):
     def error_log(self, msg="", level=20, traceback=False):
         if traceback:
@@ -127,16 +136,33 @@ def port_is_available_on_host(host, port):
 
 
 class ServerPlugin(BaseServerPlugin):
-    def __init__(self, *args, **kwargs):
-        httpserver = kwargs.get("httpserver")
-        if not isinstance(httpserver, BaseServer):
-            raise TypeError("httpserver must be a cheroot.wsgi.BaseServer")
+    def __init__(self, bus, port):
         # The server is initialized without a bind address before it is passed into the plugin.
         # Because of the property setter below for `bind_addr` the setting of `self.bind_addr`
         # in the super invocation here, results in the httpserver's `bind_addr` property being set.
-        super(ServerPlugin, self).__init__(*args, **kwargs)
+        address = (conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"], port)
+
+        super(ServerPlugin, self).__init__(
+            bus,
+            httpserver=Server(None, self.application, **self.server_config),
+            bind_addr=address,
+        )
         self._default_bind_addr = self.bind_addr
 
+    @property
+    def application(self):
+        raise NotImplementedError("ServerPlugin subclasses must implement application")
+
+    @property
+    def server_config(self):
+        return {
+            "numthreads": conf.OPTIONS["Server"]["CHERRYPY_THREAD_POOL"],
+            "request_queue_size": conf.OPTIONS["Server"]["CHERRYPY_QUEUE_SIZE"],
+            "timeout": conf.OPTIONS["Server"]["CHERRYPY_SOCKET_TIMEOUT"],
+            "accepted_queue_size": conf.OPTIONS["Server"]["CHERRYPY_QUEUE_SIZE"],
+            "accepted_queue_timeout": conf.OPTIONS["Server"]["CHERRYPY_QUEUE_TIMEOUT"],
+        }
+
     @property
     def bind_addr(self):
         # Instead of using our own copy of bind_addr, mirror httpserver.
@@ -166,6 +192,30 @@ def interface(self):
 
 
 class KolibriServerPlugin(ServerPlugin):
+    def __init__(self, bus, port):
+        # Check if there are other kolibri instances running
+        # If there are, then we need to stop users from starting kolibri again.
+        pid, port, _, status = _read_pid_file(bus.pid_file)
+        if (
+            status in IS_RUNNING
+            and pid_exists(pid)
+            and not port_is_available_on_host(
+                conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"], port
+            )
+        ):
+            logger.error(
+                "There is another Kolibri server running. "
+                "Please use `kolibri stop` and try again."
+            )
+            raise RunningException("There is another Kolibri server running.")
+        super(KolibriServerPlugin, self).__init__(bus, port)
+
+    @property
+    def application(self):
+        from kolibri.deployment.default.wsgi import application
+
+        return application
+
     def ENTER(self):
         super(KolibriServerPlugin, self).ENTER()
         # Clear old sessions up
@@ -183,6 +233,12 @@ def START(self):
 
 
 class ZipContentServerPlugin(ServerPlugin):
+    @property
+    def application(self):
+        from kolibri.deployment.default.alt_wsgi import alt_application
+
+        return alt_application
+
     def START(self):
         super(ZipContentServerPlugin, self).START()
         _, bind_port = self.httpserver.bind_addr
@@ -225,11 +281,22 @@ def STOP(self):
 class ZeroConfPlugin(Monitor):
     def __init__(self, bus, port):
         self.port = port
-        Monitor.__init__(self, bus, self.run, frequency=5)
+        if conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"] == "0.0.0.0":
+            # Only bother doing dynamic updates of the zeroconf service if we're bound
+            # to all available IP addresses.
+            Monitor.__init__(self, bus, self.run, frequency=5)
         self.bus.subscribe("SERVING", self.SERVING)
         self.bus.subscribe("UPDATE_ZEROCONF", self.UPDATE_ZEROCONF)
         self.broadcast = None
 
+    @property
+    def interfaces(self):
+        return (
+            InterfaceChoice.All
+            if conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"] == "0.0.0.0"
+            else [conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"]]
+        )
+
     def SERVING(self, port):
         # Register the Kolibri zeroconf service so it will be discoverable on the network
         from kolibri.core.discovery.utils.network.broadcast import (
@@ -246,13 +313,15 @@ def SERVING(self, port):
         instance = build_broadcast_instance(port)
 
         if self.broadcast is None:
-            self.broadcast = KolibriBroadcast(instance)
+            self.broadcast = KolibriBroadcast(instance, interfaces=self.interfaces)
             self.broadcast.add_listener(DynamicNetworkLocationListener)
             self.broadcast.add_listener(SoUDClientListener)
             self.broadcast.add_listener(SoUDServerListener)
             self.broadcast.start_broadcast()
         else:
-            self.broadcast.update_broadcast(instance=instance)
+            self.broadcast.update_broadcast(
+                instance=instance, interfaces=self.interfaces
+            )
 
     def UPDATE_ZEROCONF(self):
         self.SERVING(self.port)
@@ -277,7 +346,7 @@ def run(self):
             logger.info(
                 "List of local addresses has changed since zeroconf was last initialized, updating now"
             )
-            self.broadcast.update_broadcast(interfaces=InterfaceChoice.All)
+            self.broadcast.update_broadcast(interfaces=self.interfaces)
 
 
 status_map = {
@@ -298,6 +367,7 @@ def run(self):
 class PIDPlugin(SimplePlugin):
     def __init__(self, bus):
         self.bus = bus
+
         # Do this during initialization to set a startup lock
         self.set_pid_file(STATUS_STARTING_UP)
         self.bus.subscribe("SERVING", self.SERVING)
@@ -335,9 +405,46 @@ def EXIT(self):
             pass
 
 
+def _port_check(port):
+    # In case that something other than Kolibri occupies the port,
+    # check the port's availability.
+    # Bypass check when socket activation is used
+    # https://manpages.debian.org/testing/libsystemd-dev/sd_listen_fds.3.en.html#ENVIRONMENT
+    # Also bypass when the port is 0, as that will choose a port
+    if (
+        not os.environ.get("LISTEN_PID", None)
+        and port
+        and not port_is_available_on_host(
+            conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"], int(port)
+        )
+    ):
+        # Port is occupied
+        logger.error(
+            "Port {} is occupied.\n"
+            "Please check that you do not have other processes "
+            "running on this port and try again.\n".format(port)
+        )
+        raise PortOccupied("Port {} is occupied.".format(port))
+
+
 class DaemonizePlugin(SimplePlugin):
+    def __init__(self, bus, check_ports):
+        self.check_ports = check_ports
+        super(DaemonizePlugin, self).__init__(bus)
+
     def ENTER(self):
         self.bus.publish("log", "Running Kolibri as background process", 20)
+        if self.check_ports:
+            _port_check(self.bus.port)
+            _port_check(self.bus.zip_port)
+            if self.bus.port:
+                __, urls = get_urls(listen_port=self.bus.port)
+                for url in urls:
+                    logger.info("Kolibri running on: {}".format(url))
+            else:
+                logger.info(
+                    "No port specified, for information about accessing the server, run kolibri status"
+                )
         # Daemonize at this point, no more user output is needed
 
         kolibri_log = settings.LOGGING["handlers"]["file"]["filename"]
@@ -516,51 +623,18 @@ def stop():
     return STATUS_STOPPED
 
 
-class KolibriProcessBus(ProcessBus):
-    """
-    This class is the state machine that manages the starting, restarting, and shutdown of
-    a running Kolibri instance. It is responsible for starting any WSGI servers that respond
-    to HTTP requests in the Kolibri lifecycle, and also other ancillary services like
-    a ZeroConf server, task runner work pool, etc.
-
-    The primary use case for this process bus is for running Kolibri in a consumer server or
-    application context - although it can still be used to run the background services in
-    other contexts. One possible example for the extensibility of this class is if it is used
-    in conjunction with uwsgi, the 'restart' method of this class can be updated in a subclass
-    to run the specific `uwsgi.restart()` function that would otherwise not get invoked.
-    """
-
+class BaseKolibriProcessBus(ProcessBus):
     def __init__(
         self,
         port=0,
         zip_port=0,
-        serve_http=True,
-        background=False,
         pid_file=PID_FILE,
-        listen_address=LISTEN_ADDRESS,
     ):
-        self.listen_address = listen_address
         self.pid_file = pid_file
-        self.background = background
-        self.serve_http = serve_http
         self.port = int(port)
         self.zip_port = int(zip_port)
-        # On Mac, Python crashes when forking the process, so prevent daemonization until we can figure out
-        # a better fix. See https://github.com/learningequality/kolibri/issues/4821
-        if sys.platform == "darwin":
-            self.background = False
-
-        if (
-            self._kolibri_appears_to_be_running()
-            and self._kolibri_main_port_is_occupied()
-        ):
-            logger.error(
-                "There is another Kolibri server running. "
-                "Please use `kolibri stop` and try again."
-            )
-            sys.exit(1)
 
-        super(KolibriProcessBus, self).__init__()
+        super(BaseKolibriProcessBus, self).__init__()
         # This can be removed when a new version of magicbus is released that
         # includes their fix for Python 3.9 compatibility.
         self.thread_wait.unsubscribe()
@@ -572,27 +646,11 @@ def __init__(
         pid_plugin = PIDPlugin(self)
         pid_plugin.subscribe()
 
-        self.background_port_check()
-
         logger.info("Starting Kolibri {version}".format(version=kolibri.__version__))
 
-        if self.background:
-            daemonize_plugin = DaemonizePlugin(self)
-            daemonize_plugin.subscribe()
-
         log_plugin = LogPlugin(self)
         log_plugin.subscribe()
 
-        self.configure_http_server()
-
-        # Setup plugin for services
-        service_plugin = ServicesPlugin(self)
-        service_plugin.subscribe()
-
-        # Setup zeroconf plugin
-        zeroconf_plugin = ZeroConfPlugin(self, port)
-        zeroconf_plugin.subscribe()
-
         signal_handler = SignalHandler(self)
 
         signal_handler.subscribe()
@@ -608,106 +666,83 @@ def __init__(
         reload_plugin = ProcessControlPlugin(self)
         reload_plugin.subscribe()
 
-    def _kolibri_appears_to_be_running(self):
-        # Check if there are other kolibri instances running
-        # If there are, then we need to stop users from starting kolibri again.
-        pid, _, _, status = _read_pid_file(self.pid_file)
-        return status in IS_RUNNING and pid_exists(pid)
-
-    def _kolibri_main_port_is_occupied(self):
-        if not self.serve_http:
-            return False
-        return not port_is_available_on_host(self.listen_address, self.port)
-
-    def _port_check(self, port):
-        # In case that something other than Kolibri occupies the port,
-        # check the port's availability.
-        # Bypass check when socket activation is used
-        # https://manpages.debian.org/testing/libsystemd-dev/sd_listen_fds.3.en.html#ENVIRONMENT
-        # Also bypass when the port is 0, as that will choose a port
-        port = int(port)
-        if (
-            not os.environ.get("LISTEN_PID", None)
-            and port
-            and not port_is_available_on_host(self.listen_address, port)
-        ):
-            # Port is occupied
-            logger.error(
-                "Port {} is occupied.\n"
-                "Please check that you do not have other processes "
-                "running on this port and try again.\n".format(port)
-            )
-            sys.exit(1)
-
-    def background_port_check(self):
-        # Do this before daemonization, otherwise just let the server processes handle this
-        if self.background and self.serve_http:
-            self._port_check(self.port)
-            self._port_check(self.zip_port)
-            if self.port:
-                __, urls = get_urls(listen_port=self.port)
-                for url in urls:
-                    logger.info("Kolibri running on: {}".format(url))
-            else:
-                logger.info(
-                    "No port specified, for information about accessing the server, run kolibri status"
-                )
-
-    def configure_http_server(self):
-        if self.serve_http:
-            from kolibri.deployment.default.wsgi import application
-            from kolibri.deployment.default.alt_wsgi import alt_application
-
-            server_config = {
-                "numthreads": conf.OPTIONS["Server"]["CHERRYPY_THREAD_POOL"],
-                "request_queue_size": conf.OPTIONS["Server"]["CHERRYPY_QUEUE_SIZE"],
-                "timeout": conf.OPTIONS["Server"]["CHERRYPY_SOCKET_TIMEOUT"],
-                "accepted_queue_size": conf.OPTIONS["Server"]["CHERRYPY_QUEUE_SIZE"],
-                "accepted_queue_timeout": conf.OPTIONS["Server"][
-                    "CHERRYPY_QUEUE_TIMEOUT"
-                ],
-            }
+    def run(self):
+        self.graceful()
+        self.block()
 
-            kolibri_address = (self.listen_address, self.port)
 
-            kolibri_server = KolibriServerPlugin(
-                self,
-                httpserver=Server(None, application, **server_config),
-                bind_addr=kolibri_address,
-            )
+class KolibriServicesProcessBus(BaseKolibriProcessBus):
+    def __init__(self, *args, **kwargs):
+        super(KolibriServicesProcessBus, self).__init__(*args, **kwargs)
 
-            alt_port_addr = (
-                self.listen_address,
-                self.zip_port,
-            )
+        # Setup plugin for services
+        service_plugin = ServicesPlugin(self)
+        service_plugin.subscribe()
 
-            alt_port_server = ZipContentServerPlugin(
-                self,
-                httpserver=Server(None, alt_application, **server_config),
-                bind_addr=alt_port_addr,
-            )
-            # Subscribe these servers
-            kolibri_server.subscribe()
-            alt_port_server.subscribe()
+        # Setup zeroconf plugin
+        zeroconf_plugin = ZeroConfPlugin(self, self.port)
+        zeroconf_plugin.subscribe()
 
     def run(self):
         self.graceful()
-        if not self.serve_http:
-            self.publish("SERVING", self.port)
+        self.publish("SERVING", self.port)
 
         self.block()
 
 
+class KolibriProcessBus(KolibriServicesProcessBus):
+    """
+    This class is the state machine that manages the starting, restarting, and shutdown of
+    a running Kolibri instance. It is responsible for starting any WSGI servers that respond
+    to HTTP requests in the Kolibri lifecycle, and also other ancillary services like
+    a ZeroConf server, task runner work pool, etc.
+
+    The primary use case for this process bus is for running Kolibri in a consumer server or
+    application context - although it can still be used to run the background services in
+    other contexts. One possible example for the extensibility of this class is if it is used
+    in conjunction with uwsgi, the 'restart' method of this class can be updated in a subclass
+    to run the specific `uwsgi.restart()` function that would otherwise not get invoked.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(KolibriProcessBus, self).__init__(*args, **kwargs)
+
+        kolibri_server = KolibriServerPlugin(
+            self,
+            self.port,
+        )
+
+        alt_port_server = ZipContentServerPlugin(
+            self,
+            self.zip_port,
+        )
+        # Subscribe these servers
+        kolibri_server.subscribe()
+        alt_port_server.subscribe()
+
+
 def start(port=0, zip_port=0, serve_http=True, background=False):
     """
     Starts the server.
 
     :param: port: Port number (default: 0) - assigned by free port
     """
+    if serve_http:
+        bus = KolibriProcessBus(
+            port=port,
+            zip_port=zip_port,
+        )
+    else:
+        bus = KolibriServicesProcessBus(
+            port=port,
+            zip_port=zip_port,
+        )
 
-    bus = KolibriProcessBus(
-        port=port, zip_port=zip_port, serve_http=serve_http, background=background
-    )
+    # On Mac, Python crashes when forking the process, so prevent daemonization until we can figure out
+    # a better fix. See https://github.com/learningequality/kolibri/issues/4821
+    if background and sys.platform != "darwin":
+        daemonize_plugin = DaemonizePlugin(bus, serve_http)
+        daemonize_plugin.subscribe()
 
     bus.run()
 
@@ -783,13 +818,13 @@ def get_status():  # noqa: max-complexity=16
 
     if status == STATUS_STARTING_UP:
         try:
-            wait_for_occupied_port(LISTEN_ADDRESS, port)
+            wait_for_occupied_port(conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"], port)
         except OSError:
             raise NotRunning(STATUS_FAILED_TO_START)
 
     if status == STATUS_SHUTTING_DOWN:
         try:
-            wait_for_free_port(LISTEN_ADDRESS, port)
+            wait_for_free_port(conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"], port)
         except OSError:
             raise NotRunning(STATUS_UNCLEAN_SHUTDOWN)
         raise NotRunning(STATUS_STOPPED)
@@ -839,7 +874,11 @@ def get_status():  # noqa: max-complexity=16
         except (requests.exceptions.RequestException):
             return pid, "", ""
 
-    return pid, LISTEN_ADDRESS, listen_port  # Correct PID !
+    return (
+        pid,
+        conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"],
+        listen_port,
+    )  # Correct PID !
     # We don't detect this at present:
     # Could be detected because we fetch the PID directly via HTTP, but this
     # is dumb because kolibri could be running in a worker pool with different
@@ -863,7 +902,12 @@ def get_urls(listen_port=None):
         urls = []
         if port:
             try:
-                for ip in get_all_addresses():
+                all_addresses = (
+                    get_all_addresses()
+                    if conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"] == "0.0.0.0"
+                    else [conf.OPTIONS["Deployment"]["LISTEN_ADDRESS"]]
+                )
+                for ip in all_addresses:
                     urls.append("http://{}:{}/".format(ip, port))
             except RuntimeError:
                 logger.error("Error retrieving network interface list!")


Test patch:
diff --git a/kolibri/utils/tests/test_server.py b/kolibri/utils/tests/test_server.py
--- a/kolibri/utils/tests/test_server.py
+++ b/kolibri/utils/tests/test_server.py
@@ -228,17 +228,15 @@ def test_services_shutdown_on_stop(self, mock_kolibri_broadcast):
     "kolibri.utils.server._read_pid_file", return_value=((None, None, None, None))
 )
 class ServerInitializationTestCase(TestCase):
-    @mock.patch("kolibri.utils.server.logging.error")
+    @mock.patch("kolibri.utils.server.logger.error")
     @mock.patch("kolibri.utils.server.wait_for_free_port")
     def test_port_occupied(self, wait_for_port_mock, logging_mock, read_pid_file_mock):
         wait_for_port_mock.side_effect = OSError
-        with self.assertRaises(SystemExit):
-            process = server.KolibriProcessBus("8080", "8081")
-            process.background = True
-            process.background_port_check()
-            logging_mock.assert_called()
+        with self.assertRaises(server.PortOccupied):
+            server._port_check("8080")
+        logging_mock.assert_called()
 
-    @mock.patch("kolibri.utils.server.logging.error")
+    @mock.patch("kolibri.utils.server.logger.error")
     @mock.patch("kolibri.utils.server.wait_for_free_port")
     def test_port_occupied_socket_activation(
         self, wait_for_port_mock, logging_mock, read_pid_file_mock
@@ -246,20 +244,16 @@ def test_port_occupied_socket_activation(
         wait_for_port_mock.side_effect = OSError
         # LISTEN_PID environment variable would be set if using socket activation
         with mock.patch.dict(os.environ, {"LISTEN_PID": "1234"}):
-            process = server.KolibriProcessBus("8080", "8081")
-            process.background = True
-            process.background_port_check()
+            server._port_check("8080")
             logging_mock.assert_not_called()
 
-    @mock.patch("kolibri.utils.server.logging.error")
+    @mock.patch("kolibri.utils.server.logger.error")
     @mock.patch("kolibri.utils.server.wait_for_free_port")
     def test_port_zero_zip_port_zero(
         self, wait_for_port_mock, logging_mock, read_pid_file_mock
     ):
         wait_for_port_mock.side_effect = OSError
-        process = server.KolibriProcessBus(0, 0)
-        process.background = True
-        process.background_port_check()
+        server._port_check(0)
         logging_mock.assert_not_called()
 
     @mock.patch("kolibri.utils.server.pid_exists")
@@ -278,7 +272,7 @@ def test_server_running(
         wait_for_port_mock.side_effect = OSError
         pid_exists_mock.return_value = True
         read_pid_file_mock.return_value = (1000, 8000, 8001, server.STATUS_RUNNING)
-        with self.assertRaises(SystemExit):
+        with self.assertRaises(server.PortOccupied):
             server.start(port=8000)