Repository: zalando/patroni

Issue:
Patroni python3 watchdog problem
Centos 7, python3
Using watchdog in configuration, the supposed to be leader does not properly start. Issue can be repeated by patronictl failover too. 
```
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: Traceback (most recent call last):
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/bin/patroni", line 11, in <module>
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: sys.exit(main())
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/__init__.py", line 173, in main
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: return patroni_main()
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/__init__.py", line 142, in patroni_main
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: patroni.run()
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/__init__.py", line 114, in run
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: logger.info(self.ha.run_cycle())
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/ha.py", line 1056, in run_cycle
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: info = self._run_cycle()
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/ha.py", line 1027, in _run_cycle
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: return self.process_unhealthy_cluster()
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/ha.py", line 657, in process_unhealthy_cluster
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: 'promoted self to leader by acquiring session lock')
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/ha.py", line 345, in enforce_master_role
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: if not self.watchdog.activate():
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/watchdog/base.py", line 36, in wrapped
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: return func(self, *args, **kwargs)
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/watchdog/base.py", line 124, in activate
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: return self._activate()
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/watchdog/base.py", line 144, in _activate
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: actual_timeout = self._set_timeout()
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/watchdog/base.py", line 172, in _set_timeout
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: self.impl.set_timeout(self.config.timeout)
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/watchdog/linux.py", line 202, in set_timeout
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: File "/usr/lib/python3.4/site-packages/patroni/watchdog/linux.py", line 163, in _ioctl
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: if result < 0:
Sep 22 08:39:02 pgdron002d1vteo patroni[2707]: TypeError: unorderable types: bytes() < int()
```




Discussion:
Hi @DeathBorn,

what patroni version do you use?
Sorry. Patroni 1.3.3. The same was with 1.3.4
@ants, I think this is something for you.
Taking a look at it now. `fcntl.ioctl()` is somehow returning a `bytes` object. Python's documentation says that it is supposed to return an int...
Pythons [HTML documentation](https://docs.python.org/3/library/fcntl.html#fcntl.ioctl) and [source code](https://github.com/python/cpython/blob/master/Modules/fcntlmodule.c#L110) don't match. Fix coming up soon, I need to dig through source history to see how this has changed over the versions.

Until the fix is available the workaround is to just disable watchdog in config:

    watchdog:
      mode: off

Resolving patch:
diff --git a/features/steps/watchdog.py b/features/steps/watchdog.py
--- a/features/steps/watchdog.py
+++ b/features/steps/watchdog.py
@@ -31,7 +31,7 @@ def watchdog_was_closed(context, name):
     assert context.pctl.get_watchdog(name).was_closed
 
 
-@step('I wait for next {name:w} watchdog ping')
+@step('I reset {name:w} watchdog state')
 def watchdog_reset_pinged(context, name):
     context.pctl.get_watchdog(name).reset()
 
diff --git a/patroni/watchdog/base.py b/patroni/watchdog/base.py
--- a/patroni/watchdog/base.py
+++ b/patroni/watchdog/base.py
@@ -133,6 +133,7 @@ def _activate(self):
 
         try:
             self.impl.open()
+            actual_timeout = self._set_timeout()
         except WatchdogError as e:
             logger.warning("Could not activate %s: %s", self.impl.describe(), e)
             self.impl = NullWatchdog()
@@ -141,8 +142,6 @@ def _activate(self):
             logger.warning("Watchdog implementation can't be disabled."
                            " Watchdog will trigger after Patroni loses leader key.")
 
-        actual_timeout = self._set_timeout()
-
         if not self.impl.is_running or actual_timeout > self.config.timeout:
             if self.config.mode == MODE_REQUIRED:
                 if self.impl.is_null:
diff --git a/patroni/watchdog/linux.py b/patroni/watchdog/linux.py
--- a/patroni/watchdog/linux.py
+++ b/patroni/watchdog/linux.py
@@ -155,21 +155,25 @@ def close(self):
     def can_be_disabled(self):
         return self.get_support().has_MAGICCLOSE
 
-    def _ioctl(self, func, arg, mutate_arg=False):
+    def _ioctl(self, func, arg):
+        """Runs the specified ioctl on the underlying fd.
+
+        Raises WatchdogError if the device is closed.
+        Raises OSError or IOError (Python 2) when the ioctl fails."""
         if self._fd is None:
             raise WatchdogError("Watchdog device is closed")
-
-        result = fcntl.ioctl(self._fd, func, arg, mutate_arg)
-        if result < 0:
-            raise IOError(result)
+        fcntl.ioctl(self._fd, func, arg, True)
 
     def get_support(self):
         if self._support_cache is None:
             info = watchdog_info()
-            self._ioctl(WDIOC_GETSUPPORT, info, True)
+            try:
+                self._ioctl(WDIOC_GETSUPPORT, info)
+            except (WatchdogError, OSError, IOError) as e:
+                raise WatchdogError("Could not get information about watchdog device: {}".format(e))
             self._support_cache = WatchdogInfo(info.options,
                                                info.firmware_version,
-                                               str(bytearray(info.identity)).rstrip('\x00'))
+                                               bytearray(info.identity).decode(errors='ignore').rstrip('\x00'))
         return self._support_cache
 
     def describe(self):
@@ -180,7 +184,7 @@ def describe(self):
             try:
                 _, version, identity = self.get_support()
                 ver_str = " (firmware {0})".format(version) if version else ""
-            except WatchdogError:  # XXX: Can it really be raise when self._fd is not None?
+            except WatchdogError:
                 pass
 
         return identity + ver_str + dev_str
@@ -199,11 +203,17 @@ def set_timeout(self, timeout):
         timeout = int(timeout)
         if not 0 < timeout < 0xFFFF:
             raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout))
-        self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+        try:
+            self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+        except (WatchdogError, OSError, IOError) as e:
+            raise WatchdogError("Could not set timeout on watchdog device: {}".format(e))
 
     def get_timeout(self):
         timeout = ctypes.c_int()
-        self._ioctl(WDIOC_GETTIMEOUT, timeout, True)
+        try:
+            self._ioctl(WDIOC_GETTIMEOUT, timeout)
+        except (WatchdogError, OSError, IOError) as e:
+            raise WatchdogError("Could not get timeout on watchdog device: {}".format(e))
         return timeout.value
 
 


Test patch:
diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py
--- a/tests/test_watchdog.py
+++ b/tests/test_watchdog.py
@@ -194,15 +194,24 @@ def test_basics(self):
         self.assertRaises(WatchdogError, self.impl.set_timeout, -1)
 
     @patch('os.open', Mock(return_value=3))
-    @patch('fcntl.ioctl', Mock(return_value=-1))
+    @patch('fcntl.ioctl', Mock(side_effect=OSError))
     def test__ioctl(self):
         self.assertRaises(WatchdogError, self.impl.get_support)
         self.impl.open()
-        self.assertRaises(IOError, self.impl.get_support)
+        self.assertRaises(WatchdogError, self.impl.get_support)
 
     def test_is_healthy(self):
         self.assertFalse(self.impl.is_healthy)
 
+    @patch('os.open', Mock(return_value=3))
+    @patch('fcntl.ioctl', Mock(side_effect=OSError))
+    def test_error_handling(self):
+        self.impl.open()
+        self.assertRaises(WatchdogError, self.impl.get_timeout)
+        self.assertRaises(WatchdogError, self.impl.set_timeout, 10)
+        # We still try to output a reasonable string even if getting info errors
+        self.assertEquals(self.impl.describe(), "Linux watchdog device")
+
     @patch('os.open', Mock(side_effect=OSError))
     def test_open(self):
         self.assertRaises(WatchdogError, self.impl.open)