Repository: sql-machine-learning/elasticdl

Issue:
[Flake8] Part 5 out of 6
```
elasticdl/python/tests/worker_test.py:5:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:6:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:7:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:8:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:9:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:11:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:12:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:13:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:14:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:15:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:16:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:17:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:18:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:19:1: E402 module level import not at top of file
elasticdl/python/tests/worker_test.py:56:80: E501 line too long (83 > 79 characters)
elasticdl/python/tests/worker_test.py:62:80: E501 line too long (97 > 79 characters)
elasticdl/python/tests/worker_test.py:63:80: E501 line too long (93 > 79 characters)
elasticdl/python/data/codec/__init__.py:4:8: E225 missing whitespace around operator
elasticdl/python/elasticdl/master/k8s_worker_manager.py:121:80: E501 line too long (93 > 79 characters)
elasticdl/python/elasticdl/master/k8s_worker_manager.py:136:80: E501 line too long (82 > 79 characters)
elasticdl/python/elasticdl/master/k8s_worker_manager.py:137:80: E501 line too long (86 > 79 characters)
elasticdl/python/elasticdl/master/k8s_client.py:59:80: E501 line too long (89 > 79 characters)
elasticdl/python/elasticdl/master/k8s_client.py:60:80: E501 line too long (102 > 79 characters)
elasticdl/python/elasticdl/master/k8s_client.py:84:80: E501 line too long (83 > 79 characters)
elasticdl/python/elasticdl/master/k8s_client.py:87:80: E501 line too long (101 > 79 characters)
elasticdl/python/elasticdl/master/k8s_client.py:104:80: E501 line too long (89 > 79 characters)
elasticdl/python/elasticdl/master/k8s_client.py:105:80: E501 line too long (80 > 79 characters)
elasticdl/python/tests/checkpoint_test.py:1:1: F401 'random' imported but unused
elasticdl/python/tests/checkpoint_test.py:12:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:13:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:14:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:15:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:16:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:17:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:21:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:22:1: E402 module level import not at top of file
elasticdl/python/tests/checkpoint_test.py:23:1: E402 module level import not at top of file
```


Discussion:

Resolving patch:
diff --git a/elasticdl/python/data/codec/__init__.py b/elasticdl/python/data/codec/__init__.py
--- a/elasticdl/python/data/codec/__init__.py
+++ b/elasticdl/python/data/codec/__init__.py
@@ -1,4 +1,4 @@
 from elasticdl.python.data.codec.bytes_codec import BytesCodec
 from elasticdl.python.data.codec.tf_example_codec import TFExampleCodec
 
-__all__=[BytesCodec, TFExampleCodec]
+__all__ = [BytesCodec, TFExampleCodec]
diff --git a/elasticdl/python/elasticdl/master/k8s_client.py b/elasticdl/python/elasticdl/master/k8s_client.py
--- a/elasticdl/python/elasticdl/master/k8s_client.py
+++ b/elasticdl/python/elasticdl/master/k8s_client.py
@@ -4,14 +4,15 @@
 import traceback
 
 from kubernetes import client, config, watch
+from kubernetes.client import (
+    V1PersistentVolumeClaimVolumeSource as pvcVolumeSource,
+)
 
 WORKER_POD_NAME_PREFIX = "elasticdl-worker-"
 
 
 class Client(object):
-    def __init__(
-        self, *, worker_image, namespace, job_name, event_callback
-    ):
+    def __init__(self, *, worker_image, namespace, job_name, event_callback):
         """
         ElasticDL k8s client.
 
@@ -56,35 +57,48 @@ def _watch(self):
     def get_worker_pod_name(self, worker_id):
         return WORKER_POD_NAME_PREFIX + self._job_name + "-" + str(worker_id)
 
-    def _create_worker_pod(self, worker_id, resource_requests, resource_limits, priority,
-                           mount_path, volume_name, image_pull_policy, command, args, restart_policy):
+    def _create_worker_pod(
+        self,
+        worker_id,
+        resource_requests,
+        resource_limits,
+        priority,
+        mount_path,
+        volume_name,
+        image_pull_policy,
+        command,
+        args,
+        restart_policy,
+    ):
         # Worker container config
         container = client.V1Container(
             name=self.get_worker_pod_name(worker_id),
             image=self._image,
             command=command,
             resources=client.V1ResourceRequirements(
-                requests=resource_requests,
-                limits=resource_limits
+                requests=resource_requests, limits=resource_limits
             ),
             image_pull_policy=image_pull_policy,
-            args=args
+            args=args,
         )
 
         # Pod
         spec = client.V1PodSpec(
-            containers=[container],
-            restart_policy=restart_policy,
+            containers=[container], restart_policy=restart_policy
         )
 
         # Mount data path
         if mount_path is not None and volume_name is not None:
             volume = client.V1Volume(
-                name='data-volume',
-                persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
-                    claim_name="fileserver-claim", read_only=False))
+                name="data-volume",
+                persistent_volume_claim=pvcVolumeSource(
+                    claim_name="fileserver-claim", read_only=False
+                ),
+            )
             spec.volumes = [volume]
-            container.volume_mounts = [client.V1VolumeMount(name=volume_name, mount_path=mount_path)]
+            container.volume_mounts = [
+                client.V1VolumeMount(name=volume_name, mount_path=mount_path)
+            ]
 
         if priority is not None:
             spec.priority_class_name = priority
@@ -95,20 +109,38 @@ def _create_worker_pod(self, worker_id, resource_requests, resource_limits, prio
                 name=self.get_worker_pod_name(worker_id),
                 labels={
                     "app": "elasticdl",
-                    "elasticdl_job_name": self._job_name
+                    "elasticdl_job_name": self._job_name,
                 },
             ),
         )
         return pod
 
-    def create_worker(self, worker_id, resource_requests, resource_limits, priority=None,
-                      mount_path=None, volume_name=None, image_pull_policy=None,
-                      command=None, args=None, restart_policy="OnFailure"):
+    def create_worker(
+        self,
+        worker_id,
+        resource_requests,
+        resource_limits,
+        priority=None,
+        mount_path=None,
+        volume_name=None,
+        image_pull_policy=None,
+        command=None,
+        args=None,
+        restart_policy="OnFailure",
+    ):
         self._logger.info("Creating worker: " + str(worker_id))
         pod = self._create_worker_pod(
-            worker_id, resource_requests, resource_limits, priority,
-            mount_path, volume_name, image_pull_policy, command=command,
-            args=args, restart_policy=restart_policy)
+            worker_id,
+            resource_requests,
+            resource_limits,
+            priority,
+            mount_path,
+            volume_name,
+            image_pull_policy,
+            command=command,
+            args=args,
+            restart_policy=restart_policy,
+        )
         return self._v1.create_namespaced_pod(self._ns, pod)
 
     def delete_worker(self, worker_id):
diff --git a/elasticdl/python/elasticdl/master/k8s_worker_manager.py b/elasticdl/python/elasticdl/master/k8s_worker_manager.py
--- a/elasticdl/python/elasticdl/master/k8s_worker_manager.py
+++ b/elasticdl/python/elasticdl/master/k8s_worker_manager.py
@@ -8,33 +8,31 @@
 
 class WorkerManager(object):
     def __init__(
-            self,
-            task_q,
-            command,
-            args,
-            num_workers=1,
-            cpu_request="1000m",
-            cpu_limit="1000m",
-            memory_request="4096Mi",
-            memory_limit="4096Mi",
-            pod_priority=None,
-            mount_path=None,
-            volume_name=None,
-            image_pull_policy=None,
-            restart_policy="OnFailure",
-            **kwargs):
+        self,
+        task_q,
+        command,
+        args,
+        num_workers=1,
+        cpu_request="1000m",
+        cpu_limit="1000m",
+        memory_request="4096Mi",
+        memory_limit="4096Mi",
+        pod_priority=None,
+        mount_path=None,
+        volume_name=None,
+        image_pull_policy=None,
+        restart_policy="OnFailure",
+        **kwargs
+    ):
         self._logger = logging.getLogger(__name__)
         self._command = command
         self._args = args
         self._num_workers = num_workers
         self._resource_requests = {
             "cpu": cpu_request,
-            "memory": memory_request
-        }
-        self._resource_limits = {
-            "cpu": cpu_limit,
-            "memory": memory_limit
+            "memory": memory_request,
         }
+        self._resource_limits = {"cpu": cpu_limit, "memory": memory_limit}
         self._restart_policy = restart_policy
         self._pod_priority = pod_priority
         self._mount_path = mount_path
@@ -60,9 +58,7 @@ def __init__(
 
         self._relaunch_deleted_live_worker = True
 
-        self._k8s_client = k8s.Client(
-            event_callback=self._event_cb, **kwargs
-        )
+        self._k8s_client = k8s.Client(event_callback=self._event_cb, **kwargs)
 
     def set_relaunch_deleted_live_worker(self, val):
         self._relaunch_deleted_live_worker = bool(val)
@@ -118,7 +114,9 @@ def _event_cb(self, event):
 
         pod_name = evt_obj.metadata.name
         phase = evt_obj.status.phase
-        self._logger.info("Got event %s, phase %s for pod: %s" % (evt_type, phase, pod_name))
+        self._logger.info(
+            "Got event %s, phase %s for pod: %s" % (evt_type, phase, pod_name)
+        )
 
         relaunch = False
         with self._lock:
@@ -133,8 +131,10 @@ def _event_cb(self, event):
                 del self._pod_name_to_id[pod_name]
                 self._task_q.recover_tasks(worker_id)
 
-                # If the pod being deleted was not "Succeeded", relaunch a worker.
-                relaunch = self._relaunch_deleted_live_worker and phase != "Succeeded"
+                # If a deleted pod was not "Succeeded", relaunch a worker.
+                relaunch = (
+                    self._relaunch_deleted_live_worker and phase != "Succeeded"
+                )
         if relaunch:
             self._logger.info("Relaunching worker.")
             self._start_worker(self._next_worker_id())


Test patch:
diff --git a/elasticdl/python/tests/checkpoint_test.py b/elasticdl/python/tests/checkpoint_test.py
--- a/elasticdl/python/tests/checkpoint_test.py
+++ b/elasticdl/python/tests/checkpoint_test.py
@@ -1,12 +1,9 @@
-import random
 import os
 import tempfile
 import unittest
 import numpy as np
 import recordio
 
-import tensorflow as tf
-
 from contextlib import closing
 from elasticdl.python.elasticdl.master.servicer import MasterServicer
 from elasticdl.python.elasticdl.worker.worker import Worker
diff --git a/elasticdl/python/tests/worker_test.py b/elasticdl/python/tests/worker_test.py
--- a/elasticdl/python/tests/worker_test.py
+++ b/elasticdl/python/tests/worker_test.py
@@ -1,5 +1,3 @@
-import tensorflow as tf
-
 import tempfile
 import os
 import unittest
@@ -51,14 +49,14 @@ class _Master(InProcessMaster):
             def ReportGradient(self, req):
                 if 2 < self._m._version < 80:
                     # For testing of retrain when gradient not accepted.
-                    # Increase master version so the gradient will not be accepted.
+                    # Increase master version to reject the gradient.
                     self._m._version += 1
                 return self._m.ReportGradient(req, None)
 
             def ReportEvaluationMetrics(self, req):
                 if 2 < self._m._version < 80:
-                    # For testing of evaluation retries when evaluation metrics are not accepted.
-                    # Increase master version so the evaluation metrics will not be accepted.
+                    # Testing of evaluation retries. Increase the master
+                    # version so the evaluation metrics will not be accepted.
                     self._m._version += 1
                 return self._m.ReportEvaluationMetrics(req, None)