Repository: learningequality/kolibri

Issue:
Optimize CherryPy settings (thread_pool)
### Observed behavior

In performance testing, @bonidjukic and @jredrejo found that `server.thread_pool=30` was the worst possible setting. Incidentally that's the current setting :tada: 

 - [ ] Urgently lower `server.thread_pool` (5? 10?) - Urgent for 0.10.0?
 - [ ] React to further optimization recommendations

### Expected behavior

We use the settings that are optimal for a lower scale device and can then be tweaked by someone with more hardware resources?

### User-facing consequences

Failing requests! Request failure rate at high load is quite possible and predictable.

### Errors and logs

https://docs.google.com/spreadsheets/d/1QNHI554kXaa0CoKrHzVcb3UhfYRsjUa0X87-7Ep9AjE/edit#gid=0

https://docs.google.com/spreadsheets/u/1/d/1M77qhocswGq-U16IuY6wi3heqlUq6i3zAQOYsRCqZ_E/edit#gid=0

### Steps to reproduce

Velox



Discussion:

Resolving patch:
diff --git a/kolibri/utils/options.py b/kolibri/utils/options.py
--- a/kolibri/utils/options.py
+++ b/kolibri/utils/options.py
@@ -32,6 +32,28 @@
             "envvars": ("KOLIBRI_DATABASE_HOST",),
         },
     },
+    "Server": {
+        "CHERRYPY_THREAD_POOL": {
+            "type": "integer",
+            "default": 10,
+            "envvars": ("KOLIBRI_CHERRYPY_THREAD_POOL",),
+        },
+        "CHERRYPY_SOCKET_TIMEOUT": {
+            "type": "integer",
+            "default": 10,
+            "envvars": ("KOLIBRI_CHERRYPY_SOCKET_TIMEOUT",),
+        },
+        "CHERRYPY_QUEUE_SIZE": {
+            "type": "integer",
+            "default": 5,
+            "envvars": ("KOLIBRI_CHERRYPY_QUEUE_SIZE",),
+        },
+        "CHERRYPY_QUEUE_TIMEOUT": {
+            "type": "float",
+            "default": 0.1,
+            "envvars": ("KOLIBRI_CHERRYPY_QUEUE_TIMEOUT",),
+        },
+    },
     "Paths": {
         "CONTENT_DIR": {
             "type": "string",
diff --git a/kolibri/utils/server.py b/kolibri/utils/server.py
--- a/kolibri/utils/server.py
+++ b/kolibri/utils/server.py
@@ -12,7 +12,7 @@
 from .system import kill_pid
 from .system import pid_exists
 from kolibri.content.utils import paths
-from kolibri.utils.conf import KOLIBRI_HOME
+from kolibri.utils import conf
 
 logger = logging.getLogger(__name__)
 
@@ -30,15 +30,15 @@
 STATUS_UNKNOWN = 101
 
 # Used to store PID and port number (both in foreground and daemon mode)
-PID_FILE = os.path.join(KOLIBRI_HOME, "server.pid")
+PID_FILE = os.path.join(conf.KOLIBRI_HOME, "server.pid")
 
 # Used to PID, port during certain exclusive startup process, before we fork
 # to daemon mode
-STARTUP_LOCK = os.path.join(KOLIBRI_HOME, "server.lock")
+STARTUP_LOCK = os.path.join(conf.KOLIBRI_HOME, "server.lock")
 
 # This is a special file with daemon activity. It logs ALL stderr output, some
 # might not have made it to the log file!
-DAEMON_LOG = os.path.join(KOLIBRI_HOME, "server.log")
+DAEMON_LOG = os.path.join(conf.KOLIBRI_HOME, "server.log")
 
 # Currently non-configurable until we know how to properly handle this
 LISTEN_ADDRESS = "0.0.0.0"
@@ -151,7 +151,10 @@ def run_server(port):
     # Configure the server
     server.socket_host = LISTEN_ADDRESS
     server.socket_port = port
-    server.thread_pool = 30
+    server.thread_pool = conf.OPTIONS["Server"]["CHERRYPY_THREAD_POOL"]
+    server.socket_timeout = conf.OPTIONS["Server"]["CHERRYPY_SOCKET_TIMEOUT"]
+    server.accepted_queue_size = conf.OPTIONS["Server"]["CHERRYPY_QUEUE_SIZE"]
+    server.accepted_queue_timeout = conf.OPTIONS["Server"]["CHERRYPY_QUEUE_TIMEOUT"]
 
     # Subscribe this server
     server.subscribe()


Test patch: