Repository: dpkp/kafka-python

Issue:
KeyError: 'error_code' at kafka/protocol/struct.py __repr__
kafka-python==1.3.1, kafka 0.10.1.0
I've set consumer's session_timeout_ms bigger than group.max.session.timeout.ms in the broker and used group coordination. Here is my consumer setup:
```python
SECOND = 1000
consumer = kafka.KafkaConsumer(
    topic,
    bootstrap_servers = bootstrap_servers,
    client_id = client_id,
    group_id = group_id,
    value_deserializer = value_deserializer,
    auto_offset_reset = 'earliest',
    enable_auto_commit = False,
    consumer_timeout_ms = 1000 * 1,
    api_version = (0, 10),
    session_timeout_ms = SECOND * 60 * 20,
    heartbeat_interval_ms = SECOND,
)
```
I use celery and I got the following exception:
```
[2016-12-12 07:50:36,281: ERROR/Worker-2] Attempt to join group some_name failed due to fatal error: [Error 26] InvalidSessionTimeoutError: JoinGroupResponse_v0(error_code=26, generation_id=0, group_protocol=u'', leader_id=u'', member_id=u'', members=[])
[2016-12-12 07:50:36,464: INFO/MainProcess] Task task_name[task_id] succeeded in 0.72630248405s: None
Traceback (most recent call last):
  File "/usr/lib/python2.7/logging/__init__.py", line 851, in emit
    msg = self.format(record)
  File "/usr/lib/python2.7/logging/__init__.py", line 724, in format
    return fmt.format(record)
  File "/home/qwe/venv/local/lib/python2.7/site-packages/celery/utils/log.py", line 142, in format
    msg = logging.Formatter.format(self, record)
  File "/usr/lib/python2.7/logging/__init__.py", line 464, in format
    record.message = record.getMessage()
  File "/usr/lib/python2.7/logging/__init__.py", line 328, in getMessage
    msg = msg % self.args
  File "/home/qwe/venv/local/lib/python2.7/site-packages/kafka/protocol/struct.py", line 46, in __repr__
    key_vals.append('%s=%s' % (name, field.repr(self.__dict__[name])))
KeyError: 'error_code'
```

Discussion:

Patch:
diff --git a/kafka/coordinator/base.py b/kafka/coordinator/base.py
index e811e88..ab259dd 100644
--- a/kafka/coordinator/base.py
+++ b/kafka/coordinator/base.py
@@ -245,13 +245,12 @@ class BaseCoordinator(object):
             # ensure that there are no pending requests to the coordinator.
             # This is important in particular to avoid resending a pending
             # JoinGroup request.
-            if self._client.in_flight_request_count(self.coordinator_id):
-                while not self.coordinator_unknown():
-                    self._client.poll(delayed_tasks=False)
-                    if not self._client.in_flight_request_count(self.coordinator_id):
-                        break
-                else:
-                    continue
+            while not self.coordinator_unknown():
+                if not self._client.in_flight_request_count(self.coordinator_id):
+                    break
+                self._client.poll(delayed_tasks=False)
+            else:
+                continue
 
             future = self._send_join_group_request()
             self._client.poll(future=future)
@@ -287,6 +286,10 @@ class BaseCoordinator(object):
             e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id)
             return Future().failure(e)
 
+        elif not self._client.ready(self.coordinator_id, metadata_priority=False):
+            e = Errors.NodeNotReadyError(self.coordinator_id)
+            return Future().failure(e)
+
         # send a join group request to the coordinator
         log.info("(Re-)joining group %s", self.group_id)
         request = JoinGroupRequest[0](
@@ -417,6 +420,13 @@ class BaseCoordinator(object):
         if self.coordinator_unknown():
             e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id)
             return Future().failure(e)
+
+        # We assume that coordinator is ready if we're sending SyncGroup
+        # as it typically follows a successful JoinGroup
+        # Also note that if client.ready() enforces a metadata priority policy,
+        # we can get into an infinite loop if the leader assignment process
+        # itself requests a metadata update
+
         future = Future()
         _f = self._client.send(self.coordinator_id, request)
         _f.add_callback(self._handle_sync_group_response, future, time.time())
@@ -468,6 +478,10 @@ class BaseCoordinator(object):
         if node_id is None:
             return Future().failure(Errors.NoBrokersAvailable())
 
+        elif not self._client.ready(node_id, metadata_priority=False):
+            e = Errors.NodeNotReadyError(node_id)
+            return Future().failure(e)
+
         log.debug("Sending group coordinator request for group %s to broker %s",
                   self.group_id, node_id)
         request = GroupCoordinatorRequest[0](self.group_id)
@@ -554,6 +568,14 @@ class BaseCoordinator(object):
 
     def _send_heartbeat_request(self):
         """Send a heartbeat request"""
+        if self.coordinator_unknown():
+            e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id)
+            return Future().failure(e)
+
+        elif not self._client.ready(self.coordinator_id, metadata_priority=False):
+            e = Errors.NodeNotReadyError(self.coordinator_id)
+            return Future().failure(e)
+
         request = HeartbeatRequest[0](self.group_id, self.generation, self.member_id)
         log.debug("Heartbeat: %s[%s] %s", request.group, request.generation_id, request.member_id)  # pylint: disable-msg=no-member
         future = Future()
diff --git a/kafka/protocol/struct.py b/kafka/protocol/struct.py
index 4c1afcb..3288172 100644
--- a/kafka/protocol/struct.py
+++ b/kafka/protocol/struct.py
@@ -18,7 +18,12 @@ class Struct(AbstractType):
         elif len(args) > 0:
             raise ValueError('Args must be empty or mirror schema')
         else:
-            self.__dict__.update(kwargs)
+            for name in self.SCHEMA.names:
+                self.__dict__[name] = kwargs.pop(name, None)
+            if kwargs:
+                raise ValueError('Keyword(s) not in schema %s: %s'
+                                 % (list(self.SCHEMA.names),
+                                    ', '.join(kwargs.keys())))
 
         # overloading encode() to support both class and instance
         # Without WeakMethod() this creates circular ref, which


Test patch:
diff --git a/test/test_protocol.py b/test/test_protocol.py
index 1c9f0f9..aa3dd17 100644
--- a/test/test_protocol.py
+++ b/test/test_protocol.py
@@ -7,8 +7,9 @@ import six
 
 from kafka.protocol.api import RequestHeader
 from kafka.protocol.commit import GroupCoordinatorRequest
-from kafka.protocol.fetch import FetchResponse
+from kafka.protocol.fetch import FetchRequest, FetchResponse
 from kafka.protocol.message import Message, MessageSet, PartialMessage
+from kafka.protocol.metadata import MetadataRequest
 from kafka.protocol.types import Int16, Int32, Int64, String
 
 
@@ -244,3 +245,16 @@ def test_decode_fetch_response_partial():
     m1 = partitions[0][3]
     assert len(m1) == 2
     assert m1[1] == (None, None, PartialMessage())
+
+
+def test_struct_unrecognized_kwargs():
+    try:
+        mr = MetadataRequest[0](topicz='foo')
+        assert False, 'Structs should not allow unrecognized kwargs'
+    except ValueError:
+        pass
+
+
+def test_struct_missing_kwargs():
+    fr = FetchRequest[0](max_wait_time=100)
+    assert fr.min_bytes is None