Repository: sql-machine-learning/elasticdl

Issue:
Modify python\examples\resnet50_subclass to run on the ImageNet50
Modify python\examples\resnet50_subclass to run on the ImageNet50


Discussion:

Resolving patch:
diff --git a/elasticdl/python/examples/resnet50_subclass/resnet50_subclass.py b/elasticdl/python/examples/resnet50_subclass/resnet50_subclass.py
--- a/elasticdl/python/examples/resnet50_subclass/resnet50_subclass.py
+++ b/elasticdl/python/examples/resnet50_subclass/resnet50_subclass.py
@@ -169,8 +169,8 @@ def optimizer(lr=0.1):
 
 def input_fn(records):
     feature_description = {
-        "image": tf.io.FixedLenFeature([224, 224, 3], tf.float32),
-        "label": tf.io.FixedLenFeature([1], tf.int64),
+        "image": tf.io.FixedLenFeature([], tf.string),
+        "label": tf.io.FixedLenFeature([], tf.int64),
     }
     image_list = []
     label_list = []
@@ -178,13 +178,27 @@ def input_fn(records):
         # deserialization
         r = tf.io.parse_single_example(r, feature_description)
         label = r["label"].numpy()
-        image = r["image"].numpy()
-        # image = cv2.resize(image, (224, 224))
+        image = tf.image.resize(
+            tf.image.decode_jpeg(r["image"]),
+            [224, 224],
+            method=tf.image.ResizeMethod.BILINEAR,
+        )
+        image = image.numpy()
+
         # processing data
         image = image.astype(np.float32)
         image /= 255
         label = label.astype(np.int32)
-        image_list.append(image)
+        # label should start from 0，now we get start from 1
+        label = label - 1
+
+        # images with only 1 channel
+        if image.shape[-1] == 1:
+            image_list.append(np.repeat(image, 3, axis=-1))
+
+        elif image.shape[-1] == 3:
+            image_list.append(image)
+
         label_list.append(label)
 
     # batching
diff --git a/elasticdl/python/master/task_dispatcher.py b/elasticdl/python/master/task_dispatcher.py
--- a/elasticdl/python/master/task_dispatcher.py
+++ b/elasticdl/python/master/task_dispatcher.py
@@ -162,6 +162,12 @@ def report(self, task_id, success):
                 and self._evaluation_service is not None
             ):
                 evaluation_task_completed = True
+            else:
+                self._logger.info(
+                    "Task:%d completed, %d remaining tasks",
+                    task_id,
+                    len(self._todo) + len(self._doing),
+                )
         if evaluation_task_completed:
             self._evaluation_service.complete_task()
 


Test patch:
diff --git a/elasticdl/python/tests/example_test.py b/elasticdl/python/tests/example_test.py
--- a/elasticdl/python/tests/example_test.py
+++ b/elasticdl/python/tests/example_test.py
@@ -53,6 +53,34 @@ def create_recordio_file(size, shape):
     return temp_file.name
 
 
+def create_imagenet_recordio_file(size, shape):
+    image_size = 1
+    for s in shape:
+        image_size *= s
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    with closing(recordio.Writer(temp_file.name)) as f:
+        for _ in range(size):
+            # image: float -> uint8 -> tensor -> bytes
+            image = np.random.rand(image_size).reshape(shape).astype(np.uint8)
+            image = tf.image.encode_jpeg(tf.convert_to_tensor(value=image))
+            image = image.numpy()
+            label = np.ndarray([1], dtype=np.int64)
+            label[0] = np.random.randint(1, 11)
+            example_dict = {
+                "image": tf.train.Feature(
+                    bytes_list=tf.train.BytesList(value=[image])
+                ),
+                "label": tf.train.Feature(
+                    int64_list=tf.train.Int64List(value=[label])
+                ),
+            }
+            example = tf.train.Example(
+                features=tf.train.Features(feature=example_dict)
+            )
+            f.write(example.SerializeToString())
+    return temp_file.name
+
+
 class ExampleTest(unittest.TestCase):
     def distributed_train_and_evaluate(
         self,
@@ -61,6 +89,7 @@ def distributed_train_and_evaluate(
         model_class,
         model_params="",
         training=True,
+        dataset="",
     ):
         """
         Run distributed training and evaluation with a local master.
@@ -76,7 +105,11 @@ def distributed_train_and_evaluate(
             channel=None,
         )
 
-        shards = {create_recordio_file(128, image_shape): 128}
+        if dataset == "imagenet":
+            shards = {create_imagenet_recordio_file(32, image_shape): 32}
+        else:
+            shards = {create_recordio_file(128, image_shape): 128}
+
         if training:
             training_shards = shards
             evaluation_shards = shards
@@ -94,7 +127,7 @@ def distributed_train_and_evaluate(
         checkpoint_service = CheckpointService("", 0, 0, True)
         if training:
             evaluation_service = EvaluationService(
-                checkpoint_service, None, task_d, 0, 0, 2, False
+                checkpoint_service, None, task_d, 0, 0, 1, False
             )
         else:
             evaluation_service = EvaluationService(
@@ -189,6 +222,15 @@ def test_cifar10_subclass_evaluate(self):
             training=False,
         )
 
+    def test_resnet50_subclass_train(self):
+        self.distributed_train_and_evaluate(
+            "resnet50_subclass",
+            [224, 224, 3],
+            DEFAULT_SUBCLASS_CUSTOM_MODEL_NAME,
+            training=True,
+            dataset="imagenet",
+        )
+
     def test_resnet50_subclass_evaluate(self):
         self.distributed_train_and_evaluate(
             "resnet50_subclass",
@@ -196,6 +238,7 @@ def test_resnet50_subclass_evaluate(self):
             DEFAULT_SUBCLASS_CUSTOM_MODEL_NAME,
             model_params='num_classes=10,dtype="float32"',
             training=False,
+            dataset="imagenet",
         )