Repository: pytorch/examples

Issue:
Add DistributedSampler for validation data loader in Imagenet example
In examples/imagenet/main.py, we only set a distributed sampler (i.e., `train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)`) for `train_loader`, while neglecting setting the distributed sampler for `val_loader`. In this case, the loss and accuracy metrics of test logs are exactly the same among different GPUs as follows, leading to unnecessary computation.
How about adding DistributedSampler for both `train_sampler` and `val_loader`?

```
Test: [ 0/62]	Time 12.870 (12.870)	Loss 2.1374e+00 (2.1374e+00)	Acc@1   0.00 (  0.00)	Acc@5 100.00 (100.00)
Test: [10/62]	Time  0.461 ( 1.547)	Loss 2.1963e+00 (2.1630e+00)	Acc@1   0.00 (  0.00)	Acc@5  98.44 ( 99.72)
Test: [20/62]	Time  0.352 ( 0.969)	Loss 2.1709e+00 (2.1958e+00)	Acc@1   0.00 (  0.00)	Acc@5  98.44 ( 73.07)
Test: [30/62]	Time  0.487 ( 0.810)	Loss 2.3123e+00 (2.2959e+00)	Acc@1   0.00 (  0.00)	Acc@5  79.69 ( 62.75)
Test: [40/62]	Time  0.345 ( 0.714)	Loss 2.8967e+00 (2.3489e+00)	Acc@1   0.00 (  0.11)	Acc@5   1.56 ( 60.90)
Test: [50/62]	Time  0.485 ( 0.666)	Loss 2.4475e+00 (2.8809e+00)	Acc@1   0.00 ( 12.56)	Acc@5   0.00 ( 61.61)
Test: [60/62]	Time  0.401 ( 0.620)	Loss 2.2804e+00 (2.9429e+00)	Acc@1  18.75 ( 11.96)	Acc@5  21.88 ( 53.18)
 * Acc@1 11.924 Acc@5 52.917
Test: [ 0/62]	Time 12.978 (12.978)	Loss 2.1374e+00 (2.1374e+00)	Acc@1   0.00 (  0.00)	Acc@5 100.00 (100.00)
Test: [10/62]	Time  0.297 ( 1.554)	Loss 2.1963e+00 (2.1630e+00)	Acc@1   0.00 (  0.00)	Acc@5  98.44 ( 99.72)
Test: [20/62]	Time  0.150 ( 0.970)	Loss 2.1709e+00 (2.1958e+00)	Acc@1   0.00 (  0.00)	Acc@5  98.44 ( 73.07)
Test: [30/62]	Time  0.267 ( 0.810)	Loss 2.3123e+00 (2.2959e+00)	Acc@1   0.00 (  0.00)	Acc@5  79.69 ( 62.75)
Test: [40/62]	Time  0.145 ( 0.715)	Loss 2.8967e+00 (2.3489e+00)	Acc@1   0.00 (  0.11)	Acc@5   1.56 ( 60.90)
Test: [50/62]	Time  0.247 ( 0.666)	Loss 2.4475e+00 (2.8809e+00)	Acc@1   0.00 ( 12.56)	Acc@5   0.00 ( 61.61)
Test: [60/62]	Time  0.164 ( 0.620)	Loss 2.2804e+00 (2.9429e+00)	Acc@1  18.75 ( 11.96)	Acc@5  21.88 ( 53.18)
 * Acc@1 11.924 Acc@5 52.917
```
Distribute validation set in imagenet example
also display progress only on rank 0, on both train and val


Discussion:
I try to do this in https://github.com/triomino/examples/blob/master/imagenet/main.py. JoinableQueue is used to gather testing results because I am not familiar with other multiprocessing api.
Now the program works but I am not sure if it is a correct or good practice. I am also waiting for an official version mentioned in https://github.com/pytorch/examples/issues/461.

Update: use dist.all_reduce() to collect testing results.
Update: DistributedSampler cause incorrect validation accuracy, as mentioned in https://github.com/pytorch/pytorch/issues/25162.


Resolving patch:
diff --git a/imagenet/main.py b/imagenet/main.py
--- a/imagenet/main.py
+++ b/imagenet/main.py
@@ -18,6 +18,7 @@
 import torchvision.transforms as transforms
 import torchvision.datasets as datasets
 import torchvision.models as models
+from torch.utils.data import Subset
 
 model_names = sorted(name for name in models.__dict__
     if name.islower() and not name.startswith("__")
@@ -214,24 +215,29 @@ def main_worker(gpu, ngpus_per_node, args):
             normalize,
         ]))
 
+    val_dataset = datasets.ImageFolder(
+        valdir,
+        transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            normalize,
+        ]))
+
     if args.distributed:
         train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
+        val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset, shuffle=False, drop_last=True)
     else:
         train_sampler = None
+        val_sampler = None
 
     train_loader = torch.utils.data.DataLoader(
         train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
         num_workers=args.workers, pin_memory=True, sampler=train_sampler)
 
     val_loader = torch.utils.data.DataLoader(
-        datasets.ImageFolder(valdir, transforms.Compose([
-            transforms.Resize(256),
-            transforms.CenterCrop(224),
-            transforms.ToTensor(),
-            normalize,
-        ])),
-        batch_size=args.batch_size, shuffle=False,
-        num_workers=args.workers, pin_memory=True)
+        val_dataset, batch_size=args.batch_size, shuffle=False,
+        num_workers=args.workers, pin_memory=True, sampler=val_sampler)
 
     if args.evaluate:
         validate(val_loader, model, criterion, args)
@@ -307,48 +313,64 @@ def train(train_loader, model, criterion, optimizer, epoch, args):
         end = time.time()
 
         if i % args.print_freq == 0:
-            progress.display(i)
+            progress.display(i + 1)
 
 
 def validate(val_loader, model, criterion, args):
+
+    def run_validate(loader, base_progress=0):
+        with torch.no_grad():
+            end = time.time()
+            for i, (images, target) in enumerate(loader):
+                i = base_progress + i
+                if args.gpu is not None:
+                    images = images.cuda(args.gpu, non_blocking=True)
+                if torch.cuda.is_available():
+                    target = target.cuda(args.gpu, non_blocking=True)
+
+                # compute output
+                output = model(images)
+                loss = criterion(output, target)
+
+                # measure accuracy and record loss
+                acc1, acc5 = accuracy(output, target, topk=(1, 5))
+                losses.update(loss.item(), images.size(0))
+                top1.update(acc1[0], images.size(0))
+                top5.update(acc5[0], images.size(0))
+
+                # measure elapsed time
+                batch_time.update(time.time() - end)
+                end = time.time()
+
+                if i % args.print_freq == 0:
+                    progress.display(i + 1)
+
     batch_time = AverageMeter('Time', ':6.3f', Summary.NONE)
     losses = AverageMeter('Loss', ':.4e', Summary.NONE)
     top1 = AverageMeter('Acc@1', ':6.2f', Summary.AVERAGE)
     top5 = AverageMeter('Acc@5', ':6.2f', Summary.AVERAGE)
     progress = ProgressMeter(
-        len(val_loader),
+        len(val_loader) + (args.distributed and (len(val_loader.sampler) * args.world_size < len(val_loader.dataset))),
         [batch_time, losses, top1, top5],
         prefix='Test: ')
 
     # switch to evaluate mode
     model.eval()
 
-    with torch.no_grad():
-        end = time.time()
-        for i, (images, target) in enumerate(val_loader):
-            if args.gpu is not None:
-                images = images.cuda(args.gpu, non_blocking=True)
-            if torch.cuda.is_available():
-                target = target.cuda(args.gpu, non_blocking=True)
-
-            # compute output
-            output = model(images)
-            loss = criterion(output, target)
-
-            # measure accuracy and record loss
-            acc1, acc5 = accuracy(output, target, topk=(1, 5))
-            losses.update(loss.item(), images.size(0))
-            top1.update(acc1[0], images.size(0))
-            top5.update(acc5[0], images.size(0))
-
-            # measure elapsed time
-            batch_time.update(time.time() - end)
-            end = time.time()
+    run_validate(val_loader)
+    if args.distributed:
+        top1.all_reduce()
+        top5.all_reduce()
 
-            if i % args.print_freq == 0:
-                progress.display(i)
+    if args.distributed and (len(val_loader.sampler) * args.world_size < len(val_loader.dataset)):
+        aux_val_dataset = Subset(val_loader.dataset,
+                                 range(len(val_loader.sampler) * args.world_size, len(val_loader.dataset)))
+        aux_val_loader = torch.utils.data.DataLoader(
+            aux_val_dataset, batch_size=args.batch_size, shuffle=False,
+            num_workers=args.workers, pin_memory=True)
+        run_validate(aux_val_loader, len(val_loader))
 
-        progress.display_summary()
+    progress.display_summary()
 
     return top1.avg
 
@@ -384,6 +406,12 @@ def update(self, val, n=1):
         self.count += n
         self.avg = self.sum / self.count
 
+    def all_reduce(self):
+        total = torch.FloatTensor([self.sum, self.count])
+        dist.all_reduce(total, dist.ReduceOp.SUM, async_op=False)
+        self.sum, self.count = total.tolist()
+        self.avg = self.sum / self.count
+
     def __str__(self):
         fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
         return fmtstr.format(**self.__dict__)


Test patch: