Repository: benoitc/gunicorn

Issue:
Support using inotify with --reload
`--reload` is a very useful option when debugging and working on an application. However, in some situations (such as running gunicorn inside of a VM or a Docker container on Mac with host mounted directories), the current file polling method can cause high CPU load due to the overhead of dealing with a virtualized filesystem.

Many of these filesystems now have support for the kernel's inotify API which allows applications to monitor for filesytem events instead of polling. This is much less resource intensive and can improve CPU usage of the Reloader in many situations. It would be nice to either have gunicorn support inotify by default (on supporting platforms) or via an extra argument.



Discussion:

Resolving patch:
diff --git a/gunicorn/config.py b/gunicorn/config.py
--- a/gunicorn/config.py
+++ b/gunicorn/config.py
@@ -489,6 +489,19 @@ def validate_hostport(val):
     else:
         raise TypeError("Value must consist of: hostname:port")
 
+
+def validate_reloader(val):
+    if val is None:
+        val = 'default'
+
+    choices = ['poll', 'inotify', 'default']
+
+    if val not in choices:
+        raise ConfigError(
+            'Invalid reloader type. Must be one of: %s' % choices
+        )
+
+
 def get_default_config_file():
     config_path = os.path.join(os.path.abspath(os.getcwd()),
             'gunicorn.conf.py')
@@ -806,9 +819,12 @@ class Reload(Setting):
     name = "reload"
     section = 'Debugging'
     cli = ['--reload']
-    validator = validate_bool
-    action = 'store_true'
-    default = False
+    validator = validate_reloader
+    nargs = '?'
+    const = 'default'
+    default = None
+    meta = 'RELOADER_TYPE'
+
     desc = '''\
         Restart workers when code changes.
 
@@ -818,6 +834,15 @@ class Reload(Setting):
         The reloader is incompatible with application preloading. When using a
         paste configuration be sure that the server block does not import any
         application code or the reload will not work as designed.
+
+        When using this option, you can optionally specify whether you would
+        like to use file system polling or the kernel's inotify API to watch
+        for changes. Generally, inotify should be preferred if available
+        because it consumes less system resources. If no preference is given,
+        inotify will attempted with a fallback to FS polling.
+
+        Note: In order to use the inotify reloader, you must have the 'inotify'
+        package installed.
         '''
 
 
diff --git a/gunicorn/reloader.py b/gunicorn/reloader.py
--- a/gunicorn/reloader.py
+++ b/gunicorn/reloader.py
@@ -4,6 +4,7 @@
 # See the NOTICE for more information.
 
 import os
+import os.path
 import re
 import sys
 import time
@@ -51,3 +52,66 @@ def run(self):
                     if self._callback:
                         self._callback(filename)
             time.sleep(self._interval)
+
+try:
+    from inotify.adapters import Inotify
+    import inotify.constants
+    has_inotify = True
+except ImportError:
+    has_inotify = False
+
+
+class InotifyReloader():
+    def __init__(self, callback=None):
+        raise ImportError('You must have the inotify module installed to use '
+                          'the inotify reloader')
+
+if has_inotify:
+
+    class InotifyReloader(threading.Thread):
+        event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE
+                      | inotify.constants.IN_DELETE_SELF | inotify.constants.IN_MODIFY
+                      | inotify.constants.IN_MOVE_SELF | inotify.constants.IN_MOVED_FROM
+                      | inotify.constants.IN_MOVED_TO)
+
+        def __init__(self, extra_files=None, callback=None):
+            super(InotifyReloader, self).__init__()
+            self.setDaemon(True)
+            self._callback = callback
+            self._dirs = set()
+            self._watcher = Inotify()
+
+        def add_extra_file(self, filename):
+            dirname = os.path.dirname(filename)
+
+            if dirname in self._dirs:
+                return
+
+            self._watcher.add_watch(dirname, mask=self.event_mask)
+            self._dirs.add(dirname)
+
+        def get_dirs(self):
+            fnames = [
+                os.path.dirname(re.sub('py[co]$', 'py', module.__file__))
+                for module in list(sys.modules.values())
+                if hasattr(module, '__file__')
+            ]
+
+            return set(fnames)
+
+        def run(self):
+            self._dirs = self.get_dirs()
+
+            for dirname in self._dirs:
+                self._watcher.add_watch(dirname, mask=self.event_mask)
+
+            for event in self._watcher.event_gen():
+                if event is None:
+                    continue
+
+                filename = event[3]
+
+                self._callback(filename)
+
+
+preferred_reloader = InotifyReloader if has_inotify else Reloader
diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py
--- a/gunicorn/workers/base.py
+++ b/gunicorn/workers/base.py
@@ -14,7 +14,7 @@
 
 from gunicorn import util
 from gunicorn.workers.workertmp import WorkerTmp
-from gunicorn.reloader import Reloader
+from gunicorn.reloader import preferred_reloader, Reloader, InotifyReloader
 from gunicorn.http.errors import (
     InvalidHeader, InvalidHeaderName, InvalidRequestLine, InvalidRequestMethod,
     InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders,
@@ -85,18 +85,6 @@ def init_process(self):
         loop is initiated.
         """
 
-        # start the reloader
-        if self.cfg.reload:
-            def changed(fname):
-                self.log.info("Worker reloading: %s modified", fname)
-                self.alive = False
-                self.cfg.worker_int(self)
-                time.sleep(0.1)
-                sys.exit(0)
-
-            self.reloader = Reloader(callback=changed)
-            self.reloader.start()
-
         # set environment' variables
         if self.cfg.env:
             for k, v in self.cfg.env.items():
@@ -126,6 +114,25 @@ def changed(fname):
 
         self.load_wsgi()
 
+        # start the reloader
+        if self.cfg.reload:
+            def changed(fname):
+                self.log.info("Worker reloading: %s modified", fname)
+                self.alive = False
+                self.cfg.worker_int(self)
+                time.sleep(0.1)
+                sys.exit(0)
+
+            if self.cfg.reload == 'poll':
+                reloader_cls = Reloader
+            elif self.cfg.reload == 'inotify':
+                reloader_cls = InotifyReloader
+            else:
+                reloader_cls = preferred_reloader
+
+            self.reloader = reloader_cls(callback=changed)
+            self.reloader.start()
+
         self.cfg.post_worker_init(self)
 
         # Enter main run loop


Test patch: