Repository: horovod/horovod

Issue:
FYI, BatchAI added a demo of running Horovod on a gpu cluster
FYI, we have added a demo on how to run horovod on Azure Batch AI. https://github.com/Azure/BatchAI/tree/master/recipes/Horovod

Thanks,
Alex


Discussion:
Hi @AlexanderYukhanov, this is great!

Thanks for the reminder, let's use this issue as a tracking issue until this gets added to the README upon next docs update.

Resolving patch:
diff --git a/examples/keras_mnist.py b/examples/keras_mnist.py
--- a/examples/keras_mnist.py
+++ b/examples/keras_mnist.py
@@ -5,6 +5,7 @@
 from keras.layers import Dense, Dropout, Flatten
 from keras.layers import Conv2D, MaxPooling2D
 from keras import backend as K
+import math
 import tensorflow as tf
 import horovod.tensorflow as hvd
 
@@ -21,7 +22,7 @@
 num_classes = 10
 
 # Adjust number of epochs based on number of GPUs.
-epochs = int(12 / hvd.size())
+epochs = int(math.ceil(12.0 / hvd.size()))
 
 # Input image dimensions
 img_rows, img_cols = 28, 28
@@ -75,8 +76,15 @@
 # Broadcast variables from rank 0 to all other processes.
 K.get_session().run(hvd.broadcast_global_variables(0))
 
+callbacks = []
+
+# Save checkpoints only on worker 0 to prevent other workers from corrupting them.
+if hvd.rank() == 0:
+    callbacks.append(keras.callbacks.ModelCheckpoint('./checkpoint-{epoch}.h5'))
+
 model.fit(x_train, y_train,
           batch_size=batch_size,
+          callbacks=callbacks,
           epochs=epochs,
           verbose=1,
           validation_data=(x_test, y_test))
diff --git a/examples/tensorflow_mnist.py b/examples/tensorflow_mnist.py
--- a/examples/tensorflow_mnist.py
+++ b/examples/tensorflow_mnist.py
@@ -96,10 +96,15 @@ def main(_):
     config.gpu_options.allow_growth = True
     config.gpu_options.visible_device_list = str(hvd.local_rank())
 
+    # Save checkpoints only on worker 0 to prevent other workers from corrupting them.
+    checkpoint_dir = './checkpoints' if hvd.rank() == 0 else None
+
     # The MonitoredTrainingSession takes care of session initialization,
     # restoring from a checkpoint, saving to a checkpoint, and closing when done
     # or an error occurs.
-    with tf.train.SingularMonitoredSession(hooks=hooks, config=config) as mon_sess:
+    with tf.train.MonitoredTrainingSession(checkpoint_dir=checkpoint_dir,
+                                           hooks=hooks,
+                                           config=config) as mon_sess:
         while not mon_sess.should_stop():
             # Run a training step synchronously.
             image_, label_ = mnist.train.next_batch(100)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -115,8 +115,8 @@ def get_mpi_flags():
     except Exception:
         raise DistutilsPlatformError(
             '%s failed (see error below), is MPI in $PATH?\n'
-            'Note: If your version of MPI has custom command to show compilation flags, '
-            'please specify it via HOROVOD_MPICXX_SHOW environment variable.\n\n'
+            'Note: If your version of MPI has a custom command to show compilation flags, '
+            'please specify it with the HOROVOD_MPICXX_SHOW environment variable.\n\n'
             '%s' % (show_command, traceback.format_exc()))
 
 
@@ -175,7 +175,7 @@ def get_cuda_dirs(build_ext):
     except (CompileError, LinkError):
         raise DistutilsPlatformError(
             'CUDA library was not found (see error above).\n'
-            'Please specify correct CUDA location via HOROVOD_CUDA_HOME '
+            'Please specify correct CUDA location with the HOROVOD_CUDA_HOME '
             'environment variable or combination of HOROVOD_CUDA_INCLUDE and '
             'HOROVOD_CUDA_LIB environment variables.\n\n'
             'HOROVOD_CUDA_HOME - path where CUDA include and lib directories can be found\n'
@@ -217,7 +217,7 @@ def get_nccl_dirs(build_ext, cuda_include_dirs, cuda_lib_dirs):
     except (CompileError, LinkError):
         raise DistutilsPlatformError(
             'NCCL 2.0 library or its later version was not found (see error above).\n'
-            'Please specify correct NCCL location via HOROVOD_NCCL_HOME '
+            'Please specify correct NCCL location with the HOROVOD_NCCL_HOME '
             'environment variable or combination of HOROVOD_NCCL_INCLUDE and '
             'HOROVOD_NCCL_LIB environment variables.\n\n'
             'HOROVOD_NCCL_HOME - path where NCCL include and lib directories can be found\n'


Test patch: