Repository: horovod/horovod

Issue:
Update ML frameworks of released Docker images for v0.26.0
Before releasing the next version, versions in `docker/*/Dokerfile` should be updated to latest versions.


Discussion:

Resolving patch:
diff --git a/.github/gen-workflow-ci.py b/.github/gen-workflow-ci.py
--- a/.github/gen-workflow-ci.py
+++ b/.github/gen-workflow-ci.py
@@ -635,14 +635,14 @@ def publish_docker_images(needs: List[str], images: List[str]) -> str:
                 f'            type=semver,pattern={{{{major}}}}\n'
                 f'            type=sha\n'
                 f'\n'
-                f'      - name: Set up QEMU\n'
-                f'        uses: docker/setup-qemu-action@v1\n'
                 f'      - name: Set up Docker Buildx\n'
-                f'        uses: docker/setup-buildx-action@v1\n'
+                f'        uses: docker/setup-buildx-action@v2\n'
+                f'        with:\n'
+                f'          driver: docker\n'
                 f'\n'
                 f'      - name: Login to DockerHub\n'
                 f'        if: needs.docker-config.outputs.push == \'true\'\n'
-                f'        uses: docker/login-action@v1\n'
+                f'        uses: docker/login-action@v2\n'
                 f'        with:\n'
                 f'          username: ${{{{ secrets.DOCKERHUB_USERNAME }}}}\n'
                 f'          password: ${{{{ secrets.DOCKERHUB_TOKEN }}}}\n'
@@ -673,8 +673,43 @@ def publish_docker_images(needs: List[str], images: List[str]) -> str:
                 f'          df -h\n'
                 f'          echo ::endgroup::\n'
                 f'\n'
-                f'      - name: Build and push\n'
-                f'        uses: docker/build-push-action@v2\n'
+                f'      - name: Build image\n'
+                f'        id: build\n'
+                f'        uses: docker/build-push-action@v3\n'
+                f'        timeout-minutes: 60\n'
+                f'        with:\n'
+                f'          context: .\n'
+                f'          file: ./docker/${{{{ matrix.docker-image }}}}/Dockerfile\n'
+                f'          pull: true\n'
+                f'          push: false\n'
+                f'          load: true\n'
+                f'          tags: horovod-test\n' +
+                f'          outputs: type=docker\n' +
+                f'\n'
+                f'      - name: List image\n'
+                f'        run: |\n'
+                f'          docker image ls horovod-test\n' +
+                f'\n'
+                f'      - name: Prepare container for test\n'
+                f'        run: |\n'
+                f'          grep "RUN sed" Dockerfile.test.cpu | sed "s/^RUN //" | docker run -i --name horovod-test horovod-test:latest /bin/bash\n' +
+                ''.join([
+                    f'\n' +
+                    f"      - name: Test image ({framework} {comm})\n" +
+                    f'        if: always() && steps.build.outcome == \'success\'' + (' && matrix.docker-image != \'horovod-ray\'' if comm == 'mpi' or 'mxnet' in example else '') + '\n' +
+                    f'        run: |\n' +
+                    f'          docker start -ai horovod-test <<<"{example}"\n'
+                    for comm in ['gloo', 'mpi']
+                    for example in ([
+                        f'python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication {comm}',
+                        f'python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 {comm}',
+                    ])
+                    for framework in [re.sub('\/.*', '', re.sub('.*\/examples\/', '', example))]
+                ]) +
+                f'\n'
+                f'      - name: Push image\n'
+                f'        if: needs.docker-config.outputs.push == \'true\'\n'
+                f'        uses: docker/build-push-action@v3\n'
                 f'        timeout-minutes: 60\n'
                 f'        with:\n'
                 f'          context: .\n'
diff --git a/examples/mxnet/mxnet2_mnist.py b/examples/mxnet/mxnet2_mnist.py
--- a/examples/mxnet/mxnet2_mnist.py
+++ b/examples/mxnet/mxnet2_mnist.py
@@ -26,6 +26,10 @@
                     help='SGD momentum (default: 0.9)')
 parser.add_argument('--no-cuda', action='store_true', default=False,
                     help='disable training on GPU (default: False)')
+parser.add_argument('--fp16-allreduce', action='store_true', default=False,
+                    help='use fp16 compression during allreduce')
+parser.add_argument('--gradient-predivide-factor', type=float, default=1.0,
+                    help='apply gradient predivide factor in optimizer (default: 1.0)')
 
 # Arguments when not run through horovodrun
 parser.add_argument('--num-proc', type=int)
@@ -123,7 +127,9 @@ def evaluate(model, data_iter, context):
         hvd.broadcast_parameters(params, root_rank=0)
 
     # Horovod: create DistributedTrainer, a subclass of gluon.Trainer
-    trainer = hvd.DistributedTrainer(params, opt)
+    compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none
+    trainer = hvd.DistributedTrainer(params, opt, compression=compression,
+                                     gradient_predivide_factor=args.gradient_predivide_factor)
 
     # Create loss function and train metric
     loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()
diff --git a/examples/mxnet/mxnet_mnist.py b/examples/mxnet/mxnet_mnist.py
--- a/examples/mxnet/mxnet_mnist.py
+++ b/examples/mxnet/mxnet_mnist.py
@@ -1,14 +1,16 @@
 import argparse
 import logging
 import os
-import zipfile
 import time
+import zipfile
 
 import mxnet as mx
-import horovod.mxnet as hvd
-from mxnet import autograd, gluon, nd
+from mxnet import autograd, gluon
 from mxnet.test_utils import download
 
+import horovod
+import horovod.mxnet as hvd
+
 # Training settings
 parser = argparse.ArgumentParser(description='MXNet MNIST Example')
 
@@ -28,151 +30,170 @@
                     help='use fp16 compression during allreduce')
 parser.add_argument('--gradient-predivide-factor', type=float, default=1.0,
                     help='apply gradient predivide factor in optimizer (default: 1.0)')
-args = parser.parse_args()
-
-if not args.no_cuda:
-    # Disable CUDA if there are no GPUs.
-    if not mx.test_utils.list_gpus():
-        args.no_cuda = True
-
-logging.basicConfig(level=logging.INFO)
-logging.info(args)
-
-
-# Function to get mnist iterator given a rank
-def get_mnist_iterator(rank):
-    data_dir = "data-%d" % rank
-    if not os.path.isdir(data_dir):
-        os.makedirs(data_dir)
-    zip_file_path = download('http://data.mxnet.io/mxnet/data/mnist.zip',
-                             dirname=data_dir)
-    with zipfile.ZipFile(zip_file_path) as zf:
-        zf.extractall(data_dir)
-
-    input_shape = (1, 28, 28)
-    batch_size = args.batch_size
-
-    train_iter = mx.io.MNISTIter(
-        image="%s/train-images-idx3-ubyte" % data_dir,
-        label="%s/train-labels-idx1-ubyte" % data_dir,
-        input_shape=input_shape,
-        batch_size=batch_size,
-        shuffle=True,
-        flat=False,
-        num_parts=hvd.size(),
-        part_index=hvd.rank()
-    )
-
-    val_iter = mx.io.MNISTIter(
-        image="%s/t10k-images-idx3-ubyte" % data_dir,
-        label="%s/t10k-labels-idx1-ubyte" % data_dir,
-        input_shape=input_shape,
-        batch_size=batch_size,
-        flat=False,
-    )
-
-    return train_iter, val_iter
-
-
-# Function to define neural network
-def conv_nets():
-    net = gluon.nn.HybridSequential()
-    with net.name_scope():
-        net.add(gluon.nn.Conv2D(channels=20, kernel_size=5, activation='relu'))
-        net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
-        net.add(gluon.nn.Conv2D(channels=50, kernel_size=5, activation='relu'))
-        net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
-        net.add(gluon.nn.Flatten())
-        net.add(gluon.nn.Dense(512, activation="relu"))
-        net.add(gluon.nn.Dense(10))
-    return net
-
-
-# Function to evaluate accuracy for a model
-def evaluate(model, data_iter, context):
-    data_iter.reset()
-    metric = mx.metric.Accuracy()
-    for _, batch in enumerate(data_iter):
-        data = batch.data[0].as_in_context(context)
-        label = batch.label[0].as_in_context(context)
-        output = model(data.astype(args.dtype, copy=False))
-        metric.update([label], [output])
-
-    return metric.get()
-
-
-# Initialize Horovod
-hvd.init()
-
-# Horovod: pin context to local rank
-context = mx.cpu(hvd.local_rank()) if args.no_cuda else mx.gpu(hvd.local_rank())
-num_workers = hvd.size()
-
-# Load training and validation data
-train_data, val_data = get_mnist_iterator(hvd.rank())
-
-# Build model
-model = conv_nets()
-model.cast(args.dtype)
-model.hybridize()
-
-# Create optimizer
-optimizer_params = {'momentum': args.momentum,
-                    'learning_rate': args.lr * hvd.size()}
-opt = mx.optimizer.create('sgd', **optimizer_params)
-
-# Initialize parameters
-initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="in",
-                             magnitude=2)
-model.initialize(initializer, ctx=context)
-
-# Horovod: fetch and broadcast parameters
-params = model.collect_params()
-if params is not None:
-    hvd.broadcast_parameters(params, root_rank=0)
-
-# Horovod: create DistributedTrainer, a subclass of gluon.Trainer
-compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none
-trainer = hvd.DistributedTrainer(params, opt, compression=compression,
-                                 gradient_predivide_factor=args.gradient_predivide_factor)
-
-# Create loss function and train metric
-loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()
-metric = mx.metric.Accuracy()
-
-# Train model
-for epoch in range(args.epochs):
-    tic = time.time()
-    train_data.reset()
-    metric.reset()
-    for nbatch, batch in enumerate(train_data, start=1):
-        data = batch.data[0].as_in_context(context)
-        label = batch.label[0].as_in_context(context)
-        with autograd.record():
+
+# Arguments when not run through horovodrun
+parser.add_argument('--num-proc', type=int)
+parser.add_argument('--hosts', help='hosts to run on in notation: hostname:slots[,host2:slots[,...]]')
+parser.add_argument('--communication', help='collaborative communication to use: gloo, mpi')
+
+
+def main(args):
+    # Function to get mnist iterator given a rank
+    def get_mnist_iterator(rank):
+        data_dir = "data-%d" % rank
+        if not os.path.isdir(data_dir):
+            os.makedirs(data_dir)
+        zip_file_path = download('http://data.mxnet.io/mxnet/data/mnist.zip',
+                                 dirname=data_dir)
+        with zipfile.ZipFile(zip_file_path) as zf:
+            zf.extractall(data_dir)
+
+        input_shape = (1, 28, 28)
+        batch_size = args.batch_size
+
+        train_iter = mx.io.MNISTIter(
+            image="%s/train-images-idx3-ubyte" % data_dir,
+            label="%s/train-labels-idx1-ubyte" % data_dir,
+            input_shape=input_shape,
+            batch_size=batch_size,
+            shuffle=True,
+            flat=False,
+            num_parts=hvd.size(),
+            part_index=hvd.rank()
+        )
+
+        val_iter = mx.io.MNISTIter(
+            image="%s/t10k-images-idx3-ubyte" % data_dir,
+            label="%s/t10k-labels-idx1-ubyte" % data_dir,
+            input_shape=input_shape,
+            batch_size=batch_size,
+            flat=False,
+        )
+
+        return train_iter, val_iter
+
+    # Function to define neural network
+    def conv_nets():
+        net = gluon.nn.HybridSequential()
+        with net.name_scope():
+            net.add(gluon.nn.Conv2D(channels=20, kernel_size=5, activation='relu'))
+            net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
+            net.add(gluon.nn.Conv2D(channels=50, kernel_size=5, activation='relu'))
+            net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
+            net.add(gluon.nn.Flatten())
+            net.add(gluon.nn.Dense(512, activation="relu"))
+            net.add(gluon.nn.Dense(10))
+        return net
+
+    # Function to evaluate accuracy for a model
+    def evaluate(model, data_iter, context):
+        data_iter.reset()
+        metric = mx.metric.Accuracy()
+        for _, batch in enumerate(data_iter):
+            data = batch.data[0].as_in_context(context)
+            label = batch.label[0].as_in_context(context)
             output = model(data.astype(args.dtype, copy=False))
-            loss = loss_fn(output, label)
-        loss.backward()
-        trainer.step(args.batch_size)
-        metric.update([label], [output])
-
-        if nbatch % 100 == 0:
-            name, acc = metric.get()
-            logging.info('[Epoch %d Batch %d] Training: %s=%f' %
-                         (epoch, nbatch, name, acc))
-
-    if hvd.rank() == 0:
-        elapsed = time.time() - tic
-        speed = nbatch * args.batch_size * hvd.size() / elapsed
-        logging.info('Epoch[%d]\tSpeed=%.2f samples/s\tTime cost=%f',
-                     epoch, speed, elapsed)
-
-    # Evaluate model accuracy
-    _, train_acc = metric.get()
-    name, val_acc = evaluate(model, val_data, context)
-    if hvd.rank() == 0:
-        logging.info('Epoch[%d]\tTrain: %s=%f\tValidation: %s=%f', epoch, name,
-                     train_acc, name, val_acc)
-
-    if hvd.rank() == 0 and epoch == args.epochs - 1:
-        assert val_acc > 0.96, "Achieved accuracy (%f) is lower than expected\
-                                (0.96)" % val_acc
+            metric.update([label], [output])
+
+        return metric.get()
+
+    # Initialize Horovod
+    hvd.init()
+
+    # Horovod: pin context to local rank
+    context = mx.cpu(hvd.local_rank()) if args.no_cuda else mx.gpu(hvd.local_rank())
+    num_workers = hvd.size()
+
+    # Load training and validation data
+    train_data, val_data = get_mnist_iterator(hvd.rank())
+
+    # Build model
+    model = conv_nets()
+    model.cast(args.dtype)
+    model.hybridize()
+
+    # Create optimizer
+    optimizer_params = {'momentum': args.momentum,
+                        'learning_rate': args.lr * hvd.size()}
+    opt = mx.optimizer.create('sgd', **optimizer_params)
+
+    # Initialize parameters
+    initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="in",
+                                 magnitude=2)
+    model.initialize(initializer, ctx=context)
+
+    # Horovod: fetch and broadcast parameters
+    params = model.collect_params()
+    if params is not None:
+        hvd.broadcast_parameters(params, root_rank=0)
+
+    # Horovod: create DistributedTrainer, a subclass of gluon.Trainer
+    compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none
+    trainer = hvd.DistributedTrainer(params, opt, compression=compression,
+                                     gradient_predivide_factor=args.gradient_predivide_factor)
+
+    # Create loss function and train metric
+    loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()
+    metric = mx.metric.Accuracy()
+
+    # Train model
+    for epoch in range(args.epochs):
+        tic = time.time()
+        train_data.reset()
+        metric.reset()
+        for nbatch, batch in enumerate(train_data, start=1):
+            data = batch.data[0].as_in_context(context)
+            label = batch.label[0].as_in_context(context)
+            with autograd.record():
+                output = model(data.astype(args.dtype, copy=False))
+                loss = loss_fn(output, label)
+            loss.backward()
+            trainer.step(args.batch_size)
+            metric.update([label], [output])
+
+            if nbatch % 100 == 0:
+                name, acc = metric.get()
+                logging.info('[Epoch %d Batch %d] Training: %s=%f' %
+                             (epoch, nbatch, name, acc))
+
+        if hvd.rank() == 0:
+            elapsed = time.time() - tic
+            speed = nbatch * args.batch_size * hvd.size() / elapsed
+            logging.info('Epoch[%d]\tSpeed=%.2f samples/s\tTime cost=%f',
+                         epoch, speed, elapsed)
+
+        # Evaluate model accuracy
+        _, train_acc = metric.get()
+        name, val_acc = evaluate(model, val_data, context)
+        if hvd.rank() == 0:
+            logging.info('Epoch[%d]\tTrain: %s=%f\tValidation: %s=%f', epoch, name,
+                         train_acc, name, val_acc)
+
+        if hvd.rank() == 0 and epoch == args.epochs - 1:
+            assert val_acc > 0.96, "Achieved accuracy (%f) is lower than expected\
+                                    (0.96)" % val_acc
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+
+    if not args.no_cuda:
+        # Disable CUDA if there are no GPUs.
+        if not mx.test_utils.list_gpus():
+            args.no_cuda = True
+
+    logging.basicConfig(level=logging.INFO)
+    logging.info(args)
+
+    if args.num_proc:
+        # run training through horovod.run
+        print(f'Running training through horovod.run')
+        horovod.run(main,
+                    args=(args,),
+                    np=args.num_proc,
+                    hosts=args.hosts,
+                    use_gloo=args.communication == 'gloo',
+                    use_mpi=args.communication == 'mpi')
+    else:
+        # this is running via horovodrun
+        main(args)


Test patch:
diff --git a/Dockerfile.test.cpu b/Dockerfile.test.cpu
--- a/Dockerfile.test.cpu
+++ b/Dockerfile.test.cpu
@@ -261,15 +261,15 @@ RUN sed -i "s/dataset.take(20000/dataset.take(100/" /horovod/examples/tensorflow
 RUN sed -i "s/dataset.take(10000/dataset.take(100/" /horovod/examples/tensorflow2/tensorflow2_mnist.py
 
 # Hack TensorFlow 2.0 Data Service example to be smaller.
-RUN sed -i "s/ epochs=24/ epochs=4/" /horovod/examples/tensorflow2/tensorflow2_mnist_data_service_train_fn_*_side_dispatcher.py
+RUN sed -i "s/ epochs=24/ epochs=2/" /horovod/examples/tensorflow2/tensorflow2_mnist_data_service_train_fn_*_side_dispatcher.py
 
 # Hack Keras MNIST advanced example to be smaller.
-RUN sed -i "s/'--epochs', type=int, default=24,/'--epochs', type=int, default=9,/" /horovod/examples/keras/keras_mnist_advanced.py
+RUN sed -i "s/'--epochs', type=int, default=24,/'--epochs', type=int, default=2,/" /horovod/examples/keras/keras_mnist_advanced.py
 RUN sed -i "s/model.add(Conv2D(32, kernel_size=(3, 3),/model.add(Conv2D(1, kernel_size=(3, 3),/" /horovod/examples/keras/keras_mnist_advanced.py
 RUN sed -i "s/model.add(Conv2D(64, (3, 3), activation='relu'))//" /horovod/examples/keras/keras_mnist_advanced.py
 
 # Hack TensorFlow 2.0 Keras MNIST advanced example to be smaller.
-RUN sed -i "s/epochs=24/epochs=9/" /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py
+RUN sed -i "s/epochs=24/epochs=2/" /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py
 RUN sed -i "s/tf.keras.layers.Conv2D(32, \\[3, 3\\],/tf.keras.layers.Conv2D(1, [3, 3],/" /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py
 RUN sed -i -E "s/\s+tf.keras.layers.Conv2D\(64, \\[3, 3\\], activation='relu'\),//" /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py
 
diff --git a/Dockerfile.test.gpu b/Dockerfile.test.gpu
--- a/Dockerfile.test.gpu
+++ b/Dockerfile.test.gpu
@@ -235,13 +235,13 @@ RUN sed -i "s/dataset.take(20000/dataset.take(100/" /horovod/examples/tensorflow
 RUN sed -i "s/dataset.take(10000/dataset.take(100/" /horovod/examples/tensorflow2/tensorflow2_mnist.py
 
 # Hack TensorFlow 2.0 Data Service example to be smaller.
-RUN sed -i "s/ epochs=24/ epochs=4/" /horovod/examples/tensorflow2/tensorflow2_mnist_data_service_train_fn_*_side_dispatcher.py
+RUN sed -i "s/ epochs=24/ epochs=2/" /horovod/examples/tensorflow2/tensorflow2_mnist_data_service_train_fn_*_side_dispatcher.py
 
 # Hack Keras MNIST advanced example to be smaller.
-RUN sed -i "s/'--epochs', type=int, default=24,/'--epochs', type=int, default=9,/" /horovod/examples/keras/keras_mnist_advanced.py
+RUN sed -i "s/'--epochs', type=int, default=24,/'--epochs', type=int, default=2,/" /horovod/examples/keras/keras_mnist_advanced.py
 
 # Hack TensorFlow 2.0 Keras MNIST advanced example to be smaller.
-RUN sed -i "s/epochs=24/epochs=9/" /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py
+RUN sed -i "s/epochs=24/epochs=2/" /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py
 
 # Hack PyTorch MNIST example to be smaller.
 RUN sed -i "s/'--epochs', type=int, default=10,/'--epochs', type=int, default=2,/" /horovod/examples/pytorch/pytorch_mnist.py
diff --git a/test/single/data/expected_buildkite_gpu_non_heads_pipeline.yaml b/test/single/data/expected_buildkite_gpu_non_heads_pipeline.yaml
--- a/test/single/data/expected_buildkite_gpu_non_heads_pipeline.yaml
+++ b/test/single/data/expected_buildkite_gpu_non_heads_pipeline.yaml
@@ -530,7 +530,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: Gloo MXNet MNIST (test-gpu-gloo-py3_7-tf1_15_5-keras2_2_4-torch1_8_1-mxnet1_5_1_p0-pyspark3_3_0)'
+- label: ':muscle: Gloo MXNet MNIST horovodrun (test-gpu-gloo-py3_7-tf1_15_5-keras2_2_4-torch1_8_1-mxnet1_5_1_p0-pyspark3_3_0)'
   command: horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py
   artifact_paths: "artifacts/**"
   env:
@@ -746,7 +746,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: Gloo MXNet MNIST (test-gpu-gloo-py3_8-tf2_7_3-keras2_7_0-torch1_10_2-mxnet1_7_0_p1-pyspark3_3_0)'
+- label: ':muscle: Gloo MXNet MNIST horovodrun (test-gpu-gloo-py3_8-tf2_7_3-keras2_7_0-torch1_10_2-mxnet1_7_0_p1-pyspark3_3_0)'
   command: horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py
   artifact_paths: "artifacts/**"
   env:
@@ -926,7 +926,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: Gloo MXNet MNIST (test-gpu-gloo-py3_8-tf2_8_2-keras2_8_0-torch1_11_0-mxnet1_8_0_p0-pyspark3_3_0)'
+- label: ':muscle: Gloo MXNet MNIST horovodrun (test-gpu-gloo-py3_8-tf2_8_2-keras2_8_0-torch1_11_0-mxnet1_8_0_p0-pyspark3_3_0)'
   command: horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py
   artifact_paths: "artifacts/**"
   env:
@@ -1106,7 +1106,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: Gloo MXNet MNIST (test-gpu-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
+- label: ':muscle: Gloo MXNet MNIST horovodrun (test-gpu-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
   command: horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py
   artifact_paths: "artifacts/**"
   env:
@@ -1178,7 +1178,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: MPI MXNet MNIST (test-gpu-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
+- label: ':muscle: MPI MXNet MNIST horovodrun (test-gpu-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
   command: bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py"
   artifact_paths: "artifacts/**"
   env:
@@ -1394,7 +1394,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: Gloo MXNet MNIST (test-mixed-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
+- label: ':muscle: Gloo MXNet MNIST horovodrun (test-mixed-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
   command: horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py
   artifact_paths: "artifacts/**"
   env:
@@ -1466,7 +1466,7 @@ steps:
     automatic: true
   agents:
     queue: 2x-gpu-v5111
-- label: ':muscle: MPI MXNet MNIST (test-mixed-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
+- label: ':muscle: MPI MXNet MNIST horovodrun (test-mixed-openmpi-gloo-py3_8-tf2_9_1-keras2_9_0-torch1_12_1-mxnet1_9_1-pyspark3_3_0)'
   command: bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py"
   artifact_paths: "artifacts/**"
   env: