Repository: celery/celery

Issue:
Connection to broker lost. Trying to re-establish the connection: OSError: [Errno 9] Bad file descriptor
## Checklist

- [ ] I have included the output of ``celery -A proj report`` in the issue.
      (if you are not able to do this, then at least specify the Celery
       version affected).
- [x] I have verified that the issue exists against the `master` branch of Celery.

## Software
celery==4.1.0
kombu==4.1.0
amqp==2.2.2
Python 3.6.1
broker: rabbitmq 3.6.14
result backend: redis

## Steps to reproduce
1. celery -A proj worker -Q Q1 --autoscale=10,1 -Ofair --without-gossip --without-mingle --heartbeat-interval=60 -n Q1
2. celery lost connection to broker
3. after restarting affected worker the connection is successfully re-established and the worker starts processing tasks

## Expected behavior
celery should re-establish connection to broker

## Actual behavior
celery tries to re-establish connection to broker but fails with this error message (which is repeated every second) until manually restarted:
```
[user] celery.worker.consumer.consumer WARNING 2017-12-18 00:38:27,078 consumer: 
Connection to broker lost. Trying to re-establish the connection...
Traceback (most recent call last):
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/worker/consumer/consumer.py", line 320, in start
    blueprint.start(self)
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/bootsteps.py", line 119, in start
    step.start(parent)
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/worker/consumer/consumer.py", line 596, in start
    c.loop(*c.loop_args())
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/worker/loops.py", line 47, in asynloop
    obj.controller.register_with_event_loop(hub)
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/worker/worker.py", line 217, in register_with_event_loop
    description='hub.register',
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/bootsteps.py", line 151, in send_all
    fun(parent, *args)
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/worker/components.py", line 178, in register_with_event_loop
    w.pool.register_with_event_loop(hub)
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/concurrency/prefork.py", line 134, in register_with_event_loop
    return reg(loop)
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/concurrency/asynpool.py", line 476, in register_with_event_loop
    for fd in self._fileno_to_outq]
  File "/home/user/.envs/user/lib/python3.6/site-packages/celery/concurrency/asynpool.py", line 476, in <listcomp>
    for fd in self._fileno_to_outq]
  File "/home/user/.envs/user/lib/python3.6/site-packages/kombu/async/hub.py", line 207, in add_reader
    return self.add(fds, callback, READ | ERR, args)
  File "/home/user/.envs/user/lib/python3.6/site-packages/kombu/async/hub.py", line 158, in add
    self.poller.register(fd, flags)
  File "/home/user/.envs/user/lib/python3.6/site-packages/kombu/utils/eventio.py", line 67, in register
    self._epoll.register(fd, events)
OSError: [Errno 9] Bad file descriptor
```


Discussion:

Resolving patch:
diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py
--- a/celery/concurrency/asynpool.py
+++ b/celery/concurrency/asynpool.py
@@ -482,8 +482,16 @@ def register_with_event_loop(self, hub):
         [self._track_child_process(w, hub) for w in self._pool]
         # Handle_result_event is called whenever one of the
         # result queues are readable.
-        [hub.add_reader(fd, self.handle_result_event, fd)
-         for fd in self._fileno_to_outq]
+        stale_fds = []
+        for fd in self._fileno_to_outq:
+            try:
+                hub.add_reader(fd, self.handle_result_event, fd)
+            except OSError:
+                logger.info("Encountered OSError while trying "
+                            "to access fd %s ", fd, exc_info=True)
+                stale_fds.append(fd)  # take note of stale fd
+        for fd in stale_fds:  # Remove now defunct file descriptors
+            self._fileno_to_outq.pop(fd, None)
 
         # Timers include calling maintain_pool at a regular interval
         # to be certain processes are restarted.
@@ -1057,7 +1065,7 @@ def create_process_queues(self):
         return inq, outq, synq
 
     def on_process_alive(self, pid):
-        """Called when reciving the :const:`WORKER_UP` message.
+        """Called when receiving the :const:`WORKER_UP` message.
 
         Marks the process as ready to receive work.
         """


Test patch:
diff --git a/t/unit/worker/test_worker.py b/t/unit/worker/test_worker.py
--- a/t/unit/worker/test_worker.py
+++ b/t/unit/worker/test_worker.py
@@ -10,8 +10,9 @@
 
 import pytest
 from amqp import ChannelError
-from case import Mock, patch, skip
+from case import Mock, mock, patch, skip
 from kombu import Connection
+from kombu.asynchronous import get_event_loop
 from kombu.common import QoS, ignore_errors
 from kombu.transport.base import Message
 from kombu.transport.memory import Transport
@@ -29,7 +30,7 @@
 from celery.utils.nodenames import worker_direct
 from celery.utils.serialization import pickle
 from celery.utils.timer2 import Timer
-from celery.worker import components, consumer, state
+from celery.worker import autoscale, components, consumer, state
 from celery.worker import worker as worker_module
 from celery.worker.consumer import Consumer
 from celery.worker.pidbox import gPidbox
@@ -791,6 +792,55 @@ def test_with_autoscaler(self):
         )
         assert worker.autoscaler
 
+    @pytest.mark.nothreads_not_lingering
+    @mock.sleepdeprived(module=autoscale)
+    def test_with_autoscaler_file_descriptor_safety(self):
+        # Given: a test celery worker instance with auto scaling
+        worker = self.create_worker(
+            autoscale=[10, 5], use_eventloop=True,
+            timer_cls='celery.utils.timer2.Timer',
+            threads=False,
+        )
+        # Given: This test requires a QoS defined on the worker consumer
+        worker.consumer.qos = qos = QoS(lambda prefetch_count: prefetch_count, 2)
+        qos.update()
+
+        # Given: We have started the worker pool
+        worker.pool.start()
+
+        # Then: the worker pool is the same as the autoscaler pool
+        auto_scaler = worker.autoscaler
+        assert worker.pool == auto_scaler.pool
+
+        # Given: Utilize kombu to get the global hub state
+        hub = get_event_loop()
+        # Given: Initial call the Async Pool to register events works fine
+        worker.pool.register_with_event_loop(hub)
+
+        # Create some mock queue message and read from them
+        _keep = [Mock(name='req{0}'.format(i)) for i in range(20)]
+        [state.task_reserved(m) for m in _keep]
+        auto_scaler.body()
+
+        # Simulate a file descriptor from the list is closed by the OS
+        # auto_scaler.force_scale_down(5)
+        # This actually works -- it releases the semaphore properly
+        # Same with calling .terminate() on the process directly
+        for fd, proc in worker.pool._pool._fileno_to_outq.items():
+            # however opening this fd as a file and closing it will do it
+            queue_worker_socket = open(str(fd), "w")
+            queue_worker_socket.close()
+            break  # Only need to do this once
+
+        # When: Calling again to register with event loop ...
+        worker.pool.register_with_event_loop(hub)
+
+        # Then: test did not raise "OSError: [Errno 9] Bad file descriptor!"
+
+        # Finally:  Clean up so the threads before/after fixture passes
+        worker.terminate()
+        worker.pool.terminate()
+
     def test_dont_stop_or_terminate(self):
         worker = self.app.WorkController(concurrency=1, loglevel=0)
         worker.stop()