Repository: sql-machine-learning/elasticdl

Issue:
The watch thread in k8s client may fail
The watch thread in k8s client may fail. After its failure, ElasticDL cannot relaunch PS/Worker pods in case of pod failure.

```
Exception in thread event_watcher:
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/urllib3/response.py", line 425, in _error_catcher
    yield
  File "/usr/local/lib/python3.6/dist-packages/urllib3/response.py", line 752, in read_chunked
    self._update_chunk_length()
  File "/usr/local/lib/python3.6/dist-packages/urllib3/response.py", line 682, in _update_chunk_length
    line = self._fp.fp.readline()
  File "/usr/lib/python3.6/socket.py", line 586, in readinto
    return self._sock.recv_into(b)
  File "/usr/lib/python3.6/ssl.py", line 1012, in recv_into
    return self.read(nbytes, buffer)
  File "/usr/lib/python3.6/ssl.py", line 874, in read
    return self._sslobj.read(len, buffer)
  File "/usr/lib/python3.6/ssl.py", line 631, in read
    v = self._sslobj.read(len, buffer)
ConnectionResetError: [Errno 104] Connection reset by peer

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.6/threading.py", line 864, in run
    self._target(*self._args, **self._kwargs)
  File "/elasticdl/python/common/k8s_client.py", line 89, in _watch
    for event in stream:
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/watch/watch.py", line 144, in stream
    for line in iter_resp_lines(resp):
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/watch/watch.py", line 48, in iter_resp_lines
    for seg in resp.read_chunked(decode_content=False):
  File "/usr/local/lib/python3.6/dist-packages/urllib3/response.py", line 781, in read_chunked
    self._original_response.close()
  File "/usr/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/lib/python3.6/dist-packages/urllib3/response.py", line 443, in _error_catcher
    raise ProtocolError("Connection broken: %r" % e, e)
urllib3.exceptions.ProtocolError: ("Connection broken: ConnectionResetError(104, 'Connection reset by peer')", ConnectionResetError(104, 'Connection reset by peer'))
```


Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/k8s_client.py b/elasticdl/python/common/k8s_client.py
--- a/elasticdl/python/common/k8s_client.py
+++ b/elasticdl/python/common/k8s_client.py
@@ -1,5 +1,6 @@
 import os
 import threading
+import time
 import traceback
 
 import yaml
@@ -81,16 +82,20 @@ def __init__(
             self.cluster = cluster_spec_module.cluster
 
     def _watch(self):
-        stream = watch.Watch().stream(
-            self.client.list_namespaced_pod,
-            self.namespace,
-            label_selector=ELASTICDL_JOB_KEY + "=" + self.job_name,
-        )
-        for event in stream:
+        while True:
             try:
-                self._event_cb(event)
+                stream = watch.Watch().stream(
+                    self.client.list_namespaced_pod,
+                    self.namespace,
+                    label_selector=ELASTICDL_JOB_KEY + "=" + self.job_name,
+                )
+                for event in stream:
+                    self._event_cb(event)
             except Exception:
                 traceback.print_exc()
+            # In case of any flaky issue causing exceptions, we wait for little
+            # time and retry.
+            time.sleep(5)
 
     def _get_service_address(self, service_name, port):
         return "%s.%s.svc:%d" % (service_name, self.namespace, port)


Test patch: