imagenet-benchmark

MobileNetV2/MobileNetV2-ENOT.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fae5b0822282cce7cec83d63b96af7bd12deae8e8371083b28a9bc6002e08a7d
+size 10682115

MobileNetV2/MobileNetV2-ENOT.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d39fa80cba1d431eea3009c7ae0bf506fb7e6c6c97853994329ebc03a1fc40e
+size 32641690

README.md

@@ -1,3 +1,61 @@
 ---
 license: apache-2.0
+datasets:
+- imagenet-1k
+library_name: torchvision
+pipeline_tag: image-classification
+tags:
+- onnx
+- ENOT-AutoDL
 ---
+
+# ENOT-AutoDL pruning benchmark on ImageNet-1k
+
+This repository contains models accelerated with [ENOT-AutoDL](https://pypi.org/project/enot-autodl/) framework.
+Models from [Torchvision](https://pytorch.org/vision/stable/models.html) are used as a baseline.
+Evaluation code is also based on Torchvision references.
+
+## ResNet-50
+
+| Model                     |  Latency (MMACs)  |  Accuracy (%)   |
+| ------------------------- | :---------------: | :-------------: |
+| **ResNet-50 Torchvision** |     4144.854      |     76.144      |
+| **ResNet-50 ENOT (x2)**   | 2057.615 (x2.014) | 75.482 (-0.662) |
+| **ResNet-50 ENOT (x4)**   | 867.943 (x4.775)  | 73.576 (-2.568) |
+
+## ViT-B/32
+
+| Model                    | Latency (MMACs)  |  Accuracy (%)   |
+| ------------------------ | :--------------: | :-------------: |
+| **ViT-B/32 Torchvision** |     4413.986     |     75.912      |
+| **ViT-B/32 ENOT**        | 492.232 (x8.967) | 73.718 (-2.194) |
+
+## MobileNetV2
+
+| Model                       | Latency (MMACs)  |  Accuracy (%)  |
+| --------------------------- | :--------------: | :------------: |
+| **MobileNetV2 Torchvision** |     334.227      |     71.878     |
+| **MobileNetV2 ENOT**        | 156.800 (x2.131) | 69.898 (-1.98) |
+
+# Validation
+
+To validate results, follow this steps:
+
+1. Install all required packages:
+   ```bash
+   pip install -r requrements.txt
+   ```
+1. Calculate model latency:
+   ```bash
+   python measure_mac.py --model-ckpt path/to/model.pth
+   ```
+1. Measure accuracy of ONNX model:
+   ```bash
+   python test.py --data-path path/to/imagenet --model-onnx path/to/model.onnx --batch-size 1
+   ```
+1. Measure accuracy of PyTorch (.pth) model:
+   ```bash
+   python test.py --data-path path/to/imagenet --model-ckpt path/to/model.pth
+   ```
+
+If you want to book a demo, please [contact us](enot@enot.ai).

ResNet-50/ResNet50-ENOT-x2.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f689ec182909427df72d390d425eb3b72618d4c40ae089b93b66ea14c6adf5f
+size 50666788

ResNet-50/ResNet50-ENOT-x2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0b29e2ac563332d02274d6d656379d3b0957b91b7c8b6c1b4433657d74d6e68
+size 101839301

ResNet-50/ResNet50-ENOT-x4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:387b705b1d83c844f513d7646f95138f8fcfb420e1ef0b5f8d7039e550c66b91
+size 20850032

ResNet-50/ResNet50-ENOT-x4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9b7d6ac9062b92da9b44f61ace8c62da76ce86fb0947fdb40fb449792e194a
+size 62177349

ViT-B-32/ViT-B-32-ENOT.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a4d0e19cfbeca9dca0b18aaf5c60b2d845c05a25e7641b954e90839efda63b
+size 39430730

ViT-B-32/ViT-B-32-ENOT.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92a81cef913af4012215215400049317168939624f09d76e9043aee2342af356
+size 157444613

measure_mac.py

@@ -0,0 +1,28 @@
+import argparse
+
+import torch
+from fvcore.nn import FlopCountAnalysis
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-ckpt", type=str)
+
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+
+    checkpoint = torch.load(args.model_ckpt, map_location="cpu")
+    model = checkpoint["model_ckpt"]
+    model.eval()
+
+    flops = FlopCountAnalysis(model.cpu(), torch.ones((1, 3, 224, 224)))
+    flops = flops.total()
+
+    print(f"MMACs = {flops/1e6}")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,5 @@
+torch==1.13.1
+torchvision==0.14.1
+fvcore==0.1.5.post20221221
+onnxruntime-gpu==1.15.1
+onnx==1.13.1

test.py

@@ -0,0 +1,180 @@
+import os
+
+import onnxruntime
+import torch
+import torch.utils.data
+import torchvision
+from torch import nn
+from torchvision.transforms.functional import InterpolationMode
+
+import utils
+
+
+def evaluate(
+    criterion,
+    data_loader,
+    device,
+    model=None,
+    model_onnx_path=None,
+    print_freq=100,
+    log_suffix="",
+):
+    if model_onnx_path:
+        session = onnxruntime.InferenceSession(
+            model_onnx_path, providers=["CPUExecutionProvider"]
+        )
+        input_name = session.get_inputs()[0].name
+
+    metric_logger = utils.MetricLogger(delimiter="  ")
+    header = f"Test: {log_suffix}"
+
+    num_processed_samples = 0
+    with torch.inference_mode():
+        for image, target in metric_logger.log_every(data_loader, print_freq, header):
+            target = target.to(device, non_blocking=True)
+            image = image.to(device)
+
+            if model_onnx_path:
+                # from torch to numpy (ort)
+                input_data = image.cpu().numpy()
+
+                output_data = session.run([], {input_name: input_data})[0]
+
+                # from numpy to torch
+                output = torch.from_numpy(output_data).to(device)
+            elif model:
+                output = model(image)
+
+            loss = criterion(output, target)
+
+            acc1, acc5 = utils.accuracy(output, target, topk=(1, 5))
+            # FIXME need to take into account that the datasets
+            # could have been padded in distributed setup
+            batch_size = image.shape[0]
+            metric_logger.update(loss=loss.item())
+            metric_logger.meters["acc1"].update(acc1.item(), n=batch_size)
+            metric_logger.meters["acc5"].update(acc5.item(), n=batch_size)
+            num_processed_samples += batch_size
+    # gather the stats from all processes
+
+    metric_logger.synchronize_between_processes()
+
+    print(
+        f"{header} Acc@1 {metric_logger.acc1.global_avg:.3f} Acc@5 {metric_logger.acc5.global_avg:.3f}"
+    )
+    return metric_logger.acc1.global_avg
+
+
+def load_data(valdir):
+    # Data loading code
+    print("Loading data")
+    interpolation = InterpolationMode("bilinear")
+
+    preprocessing = torchvision.transforms.Compose(
+        [
+            torchvision.transforms.Resize(256, interpolation=interpolation),
+            torchvision.transforms.CenterCrop(224),
+            torchvision.transforms.PILToTensor(),
+            torchvision.transforms.ConvertImageDtype(torch.float),
+            torchvision.transforms.Normalize(
+                mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
+            ),
+        ]
+    )
+
+    dataset_test = torchvision.datasets.ImageFolder(
+        valdir,
+        preprocessing,
+    )
+
+    print("Creating data loaders")
+    test_sampler = torch.utils.data.SequentialSampler(dataset_test)
+
+    return dataset_test, test_sampler
+
+
+def main(args):
+    print(args)
+
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    else:
+        device = torch.device("cpu")
+
+    val_dir = os.path.join(args.data_path, "val")
+    dataset_test, test_sampler = load_data(val_dir)
+
+    data_loader_test = torch.utils.data.DataLoader(
+        dataset_test,
+        batch_size=args.batch_size,
+        sampler=test_sampler,
+        num_workers=args.workers,
+        pin_memory=True,
+    )
+
+    print("Creating model")
+
+    criterion = nn.CrossEntropyLoss()
+
+    model = None
+    if args.model_ckpt:
+        checkpoint = torch.load(args.model_ckpt, map_location="cpu")
+        model = checkpoint["model_ckpt"]
+        if "model_ema" in checkpoint:
+            state_dict = {}
+            for key, value in checkpoint["model_ema"].items():
+                if not "module." in key:
+                    continue
+                state_dict[key.replace("module.", "")] = value
+            model.load_state_dict(state_dict)
+        model = model.to(device)
+
+    accuracy = evaluate(
+        model=model,
+        model_onnx_path=args.model_onnx,
+        criterion=criterion,
+        data_loader=data_loader_test,
+        device=device,
+    )
+    print(f"Model accuracy is: {accuracy}")
+
+
+def get_args_parser(add_help=True):
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="PyTorch Classification Training", add_help=add_help
+    )
+
+    parser.add_argument(
+        "--data-path", default="datasets/imagenet", type=str, help="dataset path"
+    )
+    parser.add_argument(
+        "-b",
+        "--batch-size",
+        default=32,
+        type=int,
+        help="images per gpu, the total batch size is $NGPU x batch_size",
+    )
+    parser.add_argument(
+        "-j",
+        "--workers",
+        default=16,
+        type=int,
+        metavar="N",
+        help="number of data loading workers (default: 16)",
+    )
+    parser.add_argument("--print-freq", default=10, type=int, help="print frequency")
+    parser.add_argument(
+        "--model-onnx", default="", type=str, help="path of .onnx checkpoint"
+    )
+    parser.add_argument(
+        "--model-ckpt", default="", type=str, help="path of .pth checkpoint"
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    args = get_args_parser().parse_args()
+    main(args)

utils.py

@@ -0,0 +1,208 @@
+import datetime
+import time
+from collections import defaultdict
+from collections import deque
+
+import torch
+import torch.distributed as dist
+
+
+class SmoothedValue:
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average."""
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.4f} ({global_avg:.4f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        t = reduce_across_processes([self.count, self.total])
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value,
+        )
+
+
+class MetricLogger:
+    def __init__(self, delimiter="\t"):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError(
+            f"'{type(self).__name__}' object has no attribute '{attr}'"
+        )
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append(f"{name}: {str(meter)}")
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ""
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt="{avg:.4f}")
+        data_time = SmoothedValue(fmt="{avg:.4f}")
+        space_fmt = ":" + str(len(str(len(iterable)))) + "d"
+        if torch.cuda.is_available():
+            log_msg = self.delimiter.join(
+                [
+                    header,
+                    "[{0" + space_fmt + "}/{1}]",
+                    "eta: {eta}",
+                    "{meters}",
+                    "time: {time}",
+                    "data: {data}",
+                    "max mem: {memory:.0f}",
+                ]
+            )
+        else:
+            log_msg = self.delimiter.join(
+                [
+                    header,
+                    "[{0" + space_fmt + "}/{1}]",
+                    "eta: {eta}",
+                    "{meters}",
+                    "time: {time}",
+                    "data: {data}",
+                ]
+            )
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                            memory=torch.cuda.max_memory_allocated() / MB,
+                        )
+                    )
+                else:
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                        )
+                    )
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print(f"{header} Total time: {total_time_str}")
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def reduce_across_processes(val):
+    if not is_dist_avail_and_initialized():
+        # nothing to sync, but we still convert to tensor for consistency with the distributed case.
+        return torch.tensor(val)
+
+    t = torch.tensor(val, device="cuda")
+    dist.barrier()
+    dist.all_reduce(t)
+    return t
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the accuracy over the k top predictions for the specified
+    values of k."""
+    with torch.inference_mode():
+        maxk = max(topk)
+        batch_size = target.size(0)
+        if target.ndim == 2:
+            target = target.max(dim=1)[1]
+
+        _, pred = output.topk(maxk, 1, True, True)
+        pred = pred.t()
+        correct = pred.eq(target[None])
+
+        res = []
+        for k in topk:
+            correct_k = correct[:k].flatten().sum(dtype=torch.float32)
+            res.append(correct_k * (100.0 / batch_size))
+        return res