Issue 1:
Running Celery with MongoDB as broker and results backend we're seeing these errors happening regularly:

[2012-09-07 21:15:25,643: ERROR/MainProcess] Consumer: Connection to broker lost. Trying to re-establish the connection...
Traceback (most recent call last):
File "/mnt/nimble/eggs/celery-3.0.9-py2.7.egg/celery/worker/consumer.py", line 369, in start
self.consume_messages()
File "/mnt/nimble/eggs/celery-3.0.9-py2.7.egg/celery/worker/consumer.py", line 842, in consume_messages
self.connection.drain_events(timeout=10.0)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/connection.py", line 197, in drain_events
return self.transport.drain_events(self.connection, **kwargs)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/__init__.py", line 743, in drain_events
item, channel = get(timeout=timeout)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/scheduling.py", line 42, in get
return self.fun(resource, **kwargs), resource
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/__init__.py", line 763, in _drain_channel
return channel.drain_events(timeout=timeout)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/__init__.py", line 560, in drain_events
return self._poll(self.cycle, timeout=timeout)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/__init__.py", line 290, in _poll
return cycle.get()
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/scheduling.py", line 42, in get
return self.fun(resource, **kwargs), resource
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/mongodb.py", line 52, in _get
msg = self._queue_cursors[queue].next()
File "/mnt/nimble/eggs/pymongo-2.1.1-py2.7-linux-x86_64.egg/pymongo/cursor.py", line 703, in next
if len(self.__data) or self._refresh():
File "/mnt/nimble/eggs/pymongo-2.1.1-py2.7-linux-x86_64.egg/pymongo/cursor.py", line 679, in _refresh
limit, self.__id))
File "/mnt/nimble/eggs/pymongo-2.1.1-py2.7-linux-x86_64.egg/pymongo/cursor.py", line 628, in __send_message
self.__tz_aware)
File "/mnt/nimble/eggs/pymongo-2.1.1-py2.7-linux-x86_64.egg/pymongo/helpers.py", line 95, in _unpack_response
cursor_id)
OperationFailure: cursor id '1323433598770136089' not valid at server
[2012-09-07 21:15:25,703: ERROR/MainProcess] Unrecoverable error: KeyError('ip-10-248-69-149.celery.pidbox',)
Traceback (most recent call last):
File "/mnt/nimble/eggs/celery-3.0.9-py2.7.egg/celery/worker/__init__.py", line 353, in start
component.start()
File "/mnt/nimble/eggs/celery-3.0.9-py2.7.egg/celery/worker/consumer.py", line 368, in start
self.reset_connection()
File "/mnt/nimble/eggs/celery-3.0.9-py2.7.egg/celery/worker/consumer.py", line 712, in reset_connection
self.reset_pidbox_node()
File "/mnt/nimble/eggs/celery-3.0.9-py2.7.egg/celery/worker/consumer.py", line 656, in reset_pidbox_node
callback=self.on_control)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/pidbox.py", line 68, in listen
callbacks=[callback or self.handle_message])
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/pidbox.py", line 59, in Consumer
**options)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/messaging.py", line 285, in __init__
self.revive(self.channel)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/messaging.py", line 297, in revive
self.declare()
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/messaging.py", line 307, in declare
queue.declare()
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/entity.py", line 387, in declare
self.queue_declare(nowait, passive=False)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/entity.py", line 409, in queue_declare
nowait=nowait)
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/virtual/__init__.py", line 401, in queue_declare
return queue, self._size(queue), 0
File "/mnt/nimble/eggs/kombu-2.4.5-py2.7.egg/kombu/transport/mongodb.py", line 73, in _size
return (self._queue_cursors[queue].count() -
KeyError: 'ip-10-248-69-149.celery.pidbox'

That's followed by Celery exiting with error code 0. MongoDB database is sharded cluster and Celery connects to it via mongos instance run on localhost. The error itself looks similar to timeouted cursor, but I believe that timeouts are not applicable to tailable cursors.

Celery report for our machine is here:

software -> celery:3.0.9 (Chiastic Slide) kombu:2.4.5 py:2.7.3
billiard:2.7.3.12 pymongo:2.1.1
platform -> system:Linux arch:64bit, ELF imp:CPython
loader -> celery.loaders.default.Loader
settings -> transport:mongodb results:mongodb

Issue 2:
We've started observing this problem in our production systems following major increase in traffic on the mongo backend.

I am cross posting this issue with kombu (celery/kombu#150), as it's not clear who is misbehaving.

[2012-08-01 00:30:27,163: ERROR/MainProcess] Unrecoverable error: KeyError('awfe1.example.com.celeryd.pidbox',)
Traceback (most recent call last):
File "/opt/lfsp/eggs/celery-2.4.6-py2.7.egg/celery/worker/__init__.py", line 268, in start
component.start()
File "/opt/lfsp/eggs/celery-2.4.6-py2.7.egg/celery/worker/consumer.py", line 309, in start
self.reset_connection()
File "/opt/lfsp/eggs/celery-2.4.6-py2.7.egg/celery/worker/consumer.py", line 592, in reset_connection
self.reset_pidbox_node()
File "/opt/lfsp/eggs/celery-2.4.6-py2.7.egg/celery/worker/consumer.py", line 531, in reset_pidbox_node
callback=self.on_control)
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/pidbox.py", line 65, in listen
callbacks=[callback or self.handle_message])
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/pidbox.py", line 56, in Consumer
**options)
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/messaging.py", line 285, in __init__
self.declare()
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/messaging.py", line 295, in declare
queue.declare()
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/entity.py", line 388, in declare
self.queue_declare(nowait, passive=False)
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/entity.py", line 408, in queue_declare
nowait=nowait)
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/transport/virtual/__init__.py", line 380, in queue_declare
return queue, self._size(queue), 0
File "/opt/lfsp/eggs/kombu-2.1.1-py2.7.egg/kombu/transport/mongodb.py", line 71, in _size
return (self._queue_cursors[queue].count() -
KeyError: 'awfe1.example.com.celeryd.pidbox'