Repository: pika/pika

Issue:
HeartbeatChecker is confused about heartbeat timeouts
cc @lukebakken, the fix should probably be back-ported to the 0.12 release candidate.

`HeartbeatChecker` constructor presently accepts an interval value and an `idle_count` which defaults to 2. `Connection` class instantiates `HeartbeatChecker` with `interval=hearbeat_timeout` and default `idle_count`.

So, if the connection is configured with a heartbeat timeout of 600 (10 minutes), it will pass 600 as the `interval` arg to `HeartbeatChecker`. So, `HearbeatChecker` will emit heartbeats to the broker only once every 600 seconds. And it will detect heartbeat timeout after 1200 seconds.

So, in the event that receipt of the heartbeat by the broker is slightly delayed (and in absence of any other AMQP frames from the client), the broker can erroneously conclude that connection with the client is lost and prematurely close the connection.

This is clearly not what was intended. `HeartbeatChecker` should be detecting a heartbeat timeout after 600 seconds of inactivity. And it should be sending a heartbeat to the broker more often than just once within the heartbeat timeout window.

I see two problems here:

1. Given `HeartbeatChecker`'s present interface, `Connection` should be instantiating it as`HeartbeatChecker(self, interval=float(self.params.heartbeat) / 2, idle_count=2) or something like that (how often does RabbitMQ broker send heartbeats within one heartbeat timeout interval?)
2. `HeartbeatChecker` is not abstracting the internals of heartbeat processing sufficiently. It's constructor should accept the heartbeat timeout value directly (no interval/idle_count business) and encapsulate the frequency of heartbeats internally without bleeding that detail to the `Connection`.


Comments:

Patch:
diff --git a/pika/heartbeat.py b/pika/heartbeat.py
index 4eaaa59..3c9f46f 100644
--- a/pika/heartbeat.py
+++ b/pika/heartbeat.py
@@ -12,21 +12,24 @@ class HeartbeatChecker(object):
     intervals.
 
     """
+    DEFAULT_INTERVAL = 60
     MAX_IDLE_COUNT = 2
+
     _STALE_CONNECTION = "Too Many Missed Heartbeats, No reply in %i seconds"
 
-    def __init__(self, connection, interval, idle_count=MAX_IDLE_COUNT):
+    def __init__(self, connection, interval=DEFAULT_INTERVAL, idle_count=MAX_IDLE_COUNT):
         """Create a heartbeat on connection sending a heartbeat frame every
         interval seconds.
 
         :param pika.connection.Connection: Connection object
-        :param int interval: Heartbeat check interval
-        :param int idle_count: Number of heartbeat intervals missed until the
-                               connection is considered idle and disconnects
+        :param int interval: Heartbeat check interval. Note: heartbeats will
+                             be sent at interval / 2 frequency.
 
         """
         self._connection = connection
-        self._interval = interval
+        # Note: see the following document:
+        # https://www.rabbitmq.com/heartbeats.html#heartbeats-timeout
+        self._interval = float(interval / 2)
         self._max_idle_count = idle_count
 
         # Initialize counters


Test patch:
diff --git a/tests/unit/connection_tests.py b/tests/unit/connection_tests.py
index 9f40320..438633c 100644
--- a/tests/unit/connection_tests.py
+++ b/tests/unit/connection_tests.py
@@ -536,7 +536,7 @@ class ConnectionTests(unittest.TestCase):  # pylint: disable=R0904
                                 _adapter_emit_data,
                                 method,
                                 heartbeat_checker):
-        """make sure on connection tune turns the connection params"""
+        """make sure _on_connection_tune tunes the connection params"""
         heartbeat_checker.return_value = 'hearbeat obj'
         self.connection._flush_outbound = mock.Mock()
         marshal = mock.Mock(return_value='ab')
diff --git a/tests/unit/heartbeat_tests.py b/tests/unit/heartbeat_tests.py
index c5a7ca5..eaf339f 100644
--- a/tests/unit/heartbeat_tests.py
+++ b/tests/unit/heartbeat_tests.py
@@ -55,27 +55,31 @@ class ConstructableConnection(connection.Connection):
 
 class HeartbeatTests(unittest.TestCase):
 
-    INTERVAL = 5
+    INTERVAL = 60
+    HALF_INTERVAL = INTERVAL / 2
 
     def setUp(self):
         self.mock_conn = mock.Mock(spec_set=ConstructableConnection())
         self.mock_conn.bytes_received = 100
         self.mock_conn.bytes_sent = 100
         self.mock_conn._heartbeat_checker = mock.Mock(spec=heartbeat.HeartbeatChecker)
-        self.obj = heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        self.obj = heartbeat.HeartbeatChecker(self.mock_conn)
 
     def tearDown(self):
         del self.obj
         del self.mock_conn
 
+    def test_default_initialization_interval(self):
+        self.assertEqual(self.obj._interval, self.HALF_INTERVAL)
+
     def test_default_initialization_max_idle_count(self):
         self.assertEqual(self.obj._max_idle_count, self.obj.MAX_IDLE_COUNT)
 
     def test_constructor_assignment_connection(self):
-        self.assertEqual(self.obj._connection, self.mock_conn)
+        self.assertIs(self.obj._connection, self.mock_conn)
 
     def test_constructor_assignment_heartbeat_interval(self):
-        self.assertEqual(self.obj._interval, self.INTERVAL)
+        self.assertEqual(self.obj._interval, self.HALF_INTERVAL)
 
     def test_constructor_initial_bytes_received(self):
         self.assertEqual(self.obj._bytes_received, 0)
@@ -94,7 +98,7 @@ class HeartbeatTests(unittest.TestCase):
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._setup_timer')
     def test_constructor_called_setup_timer(self, timer):
-        heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        heartbeat.HeartbeatChecker(self.mock_conn)
         timer.assert_called_once_with()
 
     def test_active_true(self):
@@ -122,13 +126,13 @@ class HeartbeatTests(unittest.TestCase):
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._close_connection')
     def test_send_and_check_not_closed(self, close_connection):
-        obj = heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        obj = heartbeat.HeartbeatChecker(self.mock_conn)
         obj.send_and_check()
         close_connection.assert_not_called()
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._close_connection')
     def test_send_and_check_missed_bytes(self, close_connection):
-        obj = heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        obj = heartbeat.HeartbeatChecker(self.mock_conn)
         obj._idle_byte_intervals = self.INTERVAL
         obj.send_and_check()
         close_connection.assert_called_once_with()
@@ -147,19 +151,19 @@ class HeartbeatTests(unittest.TestCase):
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._update_counters')
     def test_send_and_check_update_counters(self, update_counters):
-        obj = heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        obj = heartbeat.HeartbeatChecker(self.mock_conn)
         obj.send_and_check()
         update_counters.assert_called_once_with()
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._send_heartbeat_frame')
     def test_send_and_check_send_heartbeat_frame(self, send_heartbeat_frame):
-        obj = heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        obj = heartbeat.HeartbeatChecker(self.mock_conn)
         obj.send_and_check()
         send_heartbeat_frame.assert_called_once_with()
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._start_timer')
     def test_send_and_check_start_timer(self, start_timer):
-        obj = heartbeat.HeartbeatChecker(self.mock_conn, self.INTERVAL)
+        obj = heartbeat.HeartbeatChecker(self.mock_conn)
         obj.send_and_check()
         start_timer.assert_called_once_with()
 
@@ -202,7 +206,7 @@ class HeartbeatTests(unittest.TestCase):
 
     def test_setup_timer_called(self):
         self.mock_conn._adapter_add_timeout.assert_called_once_with(
-            self.INTERVAL, self.obj.send_and_check)
+            self.HALF_INTERVAL, self.obj.send_and_check)
 
     @mock.patch('pika.heartbeat.HeartbeatChecker._setup_timer')
     def test_start_timer_not_active(self, setup_timer):