init model

FPN_int.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a172eb921a5119875bd12561e658450ca4dae95c4aa2ea350dfd603cd27f14a
+size 45595505

README.md

@@ -0,0 +1,123 @@
+---
+license: apache-2.0
+tags:
+- RyzenAI
+- Image Segmentation
+- Pytorch
+- Vision
+datasets:
+- cityscape
+language:
+- en
+Metircs:
+- mIoU
+---
+
+# SemanticFPN model trained on cityscapes
+
+SemanticFPN is a conceptually simple yet effective baseline for panoptic segmentation trained on cityscapes. The method starts with Mask R-CNN with FPN and adds to it a lightweight semantic segmentation branch for dense-pixel prediction. It was introduced in the paper [Panoptic Feature Pyramid Networks in 2019](https://arxiv.org/pdf/1901.02446.pdf) by Kirillov, Alexander, et al.
+
+We develop a modified version that could be supported by [AMD Ryzen AI](https://ryzenai.docs.amd.com).
+
+
+## Model description
+
+SemanticFPN is a single network that unifies the tasks of instance segmentation and semantic segmentation. The network is designed by endowing Mask R-CNN, a popular instance segmentation method, with a semantic segmentation branch using a shared Feature Pyramid Network (FPN) backbone. This simple baseline not only remains effective for instance segmentation, but also yields a lightweight, top-performing method for semantic segmentation. It is a robust and accurate baseline for both tasks and can serve as a strong baseline for future research in panoptic segmentation.
+
+
+## Intended uses & limitations
+
+You can use the raw model for image segmentation. See the [model hub](https://huggingface.co/models?sort=trending&search=amd%2FSemanticFPN) to look for all available SemanticFPN models.
+
+
+## How to use
+
+### Installation
+
+   Follow [Ryzen AI Installation](https://ryzenai.docs.amd.com/en/latest/inst.html) to prepare the environment for Ryzen AI.
+   Run the following script to install pre-requisites for this model.
+   ```bash
+   pip install -r requirements.txt 
+   ```
+
+
+### Data Preparation (optional: for accuracy evaluation)
+
+1. Download cityscapes dataset (https://www.cityscapes-dataset.com/downloads)
+    - grundtruth folder: gtFine_trainvaltest.zip [241MB]
+    - image folder: leftImg8bit_trainvaltest.zip [11GB]
+2. Organize the dataset directory as follows:
+```Plain
+└── data
+     └── cityscapes
+          ├── leftImg8bit
+          |    ├── train
+          |    └── val
+          └── gtFine
+               ├── train
+               └── val
+```
+
+### Test & Evaluation
+
+- Code snippet from [`infer_onnx.py`](infer_onnx.py) on how to use
+```python
+    parser = argparse.ArgumentParser(description='SemanticFPN model')
+    parser.add_argument('--onnx_path', type=str, default='FPN_int.onnx')
+    parser.add_argument('--save_path', type=str, default='./data/demo_results/senmatic_results.png')
+    parser.add_argument('--input_path', type=str, default='data/cityscapes/cityscapes/leftImg8bit/test/bonn/bonn_000000_000019_leftImg8bit.png')
+    parser.add_argument('--ipu', action='store_true',
+                    help='use ipu')
+    parser.add_argument('--provider_config', type=str, default=None,
+                    help='provider config path')
+    args = parser.parse_args()
+
+    if args.ipu:
+        providers = ["VitisAIExecutionProvider"]
+        provider_options = [{"config_file": args.provider_config}]
+    else:
+        providers = ['CPUExecutionProvider']
+        provider_options = None
+
+    onnx_path = args.onnx_path
+    input_img = build_img(args)
+    session = onnxruntime.InferenceSession(onnx_path, providers=providers, provider_options=provider_options)
+    ort_input = {session.get_inputs()[0].name: input_img.cpu().numpy()}
+    ort_output = session.run(None, ort_input)[0]
+    if isinstance(ort_output, (tuple, list)):
+        ort_output = ort_output[0]
+
+    output = ort_output[0].transpose(1, 2, 0)
+    seg_pred = np.asarray(np.argmax(output, axis=2), dtype=np.uint8)
+    color_mask = colorize_mask(seg_pred)
+    color_mask.save(args.save_path)
+```
+
+ - Run inference for a single image
+  ```python
+  python infer_onnx.py --onnx_path FPN_int.onnx --input_path /Path/To/Your/Image --ipu --provider_config Path/To/vaip_config.json
+  ```
+
+ - Test accuracy of the quantized model
+  ```python
+  python test_onnx.py --onnx_path FPN_int.onnx --dataset citys --test-folder ./data/cityscapes --crop-size 256 --ipu --provider_config Path/To/vaip_config.json
+  ```
+### Performance
+
+| model | input size | FLOPs | mIoU on Cityscapes Validation|
+|-------|------------|--------------|-------|
+| SemanticFPN(ResNet18)| 256x512 | 10G | 62.9% |
+
+| model | input size | FLOPs | INT8 mIoU on Cityscapes Validation|
+|-------|------------|---------------|--------------|
+| SemanticFPN(ResNet18)| 256x512 | 10G | 62.5% |
+
+```bibtex
+@inproceedings{kirillov2019panoptic,
+  title={Panoptic feature pyramid networks},
+  author={Kirillov, Alexander and Girshick, Ross and He, Kaiming and Doll{\'a}r, Piotr},
+  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
+  pages={6399--6408},
+  year={2019}
+}
+```

datasets/__init__.py

@@ -0,0 +1,29 @@
+import warnings
+from torchvision.datasets import *
+from datasets.base import *
+from datasets.cityscapes import CitySegmentation
+
+
+datasets = {
+    'citys': CitySegmentation,
+}
+
+def get_dataset(name, **kwargs):
+    return datasets[name.lower()](**kwargs)
+
+def _make_deprecate(meth, old_name):
+    new_name = meth.__name__
+
+    def deprecated_init(*args, **kwargs):
+        return meth(*args, **kwargs)
+
+    deprecated_init.__doc__ = r"""
+    {old_name}(...)
+    .. warning::
+        This method is now deprecated in favor of :func:`torch.nn.init.{new_name}`.
+    See :func:`~torch.nn.init.{new_name}` for details.""".format(
+        old_name=old_name, new_name=new_name)
+    deprecated_init.__name__ = old_name
+    return deprecated_init
+
+get_segmentation_dataset = _make_deprecate(get_dataset, 'get_segmentation_dataset')

datasets/base.py

@@ -0,0 +1,96 @@
+import random
+import numpy as np
+from PIL import Image, ImageOps, ImageFilter
+import torch
+import torch.utils.data as data
+
+__all__ = ['BaseDataset']
+
+class BaseDataset(data.Dataset):
+    def __init__(self, root, split, mode=None, transform=None, 
+                 target_transform=None, base_size=1024, crop_size=512):
+        self.root = root
+        self.transform = transform
+        self.target_transform = target_transform
+        self.split = split
+        self.mode = mode if mode is not None else split
+        self.base_size = base_size
+        self.crop_size = crop_size
+        if self.mode == 'train':
+            print('BaseDataset: base_size {}, crop_size {}'. \
+                format(base_size, crop_size))
+
+    @property
+    def num_class(self):
+        return self.NUM_CLASS
+
+    def _val_transform(self, img, mask):
+        outsize = self.crop_size
+        short_size = outsize
+        w, h = img.size
+        if w > h:
+            oh = short_size
+            ow = int(1.0 * w * oh / h)
+        else:
+            ow = short_size
+            oh = int(1.0 * h * ow / w)
+        img = img.resize((ow, oh), Image.BILINEAR)
+        mask = mask.resize((ow, oh), Image.NEAREST)
+        # center crop
+        w, h = img.size
+        x1 = int(round((w - outsize) / 2.))
+        y1 = int(round((h - outsize) / 2.))
+        img = img.crop((x1, y1, x1+outsize, y1+outsize))
+        mask = mask.crop((x1, y1, x1+outsize, y1+outsize))
+        # final transform
+        return img, self._mask_transform(mask)
+
+    def _testval_transform(self, img, mask):
+        outsize = self.crop_size
+        short_size = outsize
+        w, h = img.size
+        if w > h:
+            oh = short_size
+            ow = int(1.0 * w * oh / h)
+        else:
+            ow = short_size
+            oh = int(1.0 * h * ow / w)
+        img = img.resize((ow, oh), Image.BILINEAR)
+        return img, self._mask_transform(mask)
+
+    def _train_transform(self, img, mask):
+        # random mirror
+        if random.random() < 0.5:
+            img = img.transpose(Image.FLIP_LEFT_RIGHT)
+            mask = mask.transpose(Image.FLIP_LEFT_RIGHT)
+        crop_size = self.crop_size
+        w, h = img.size
+        long_size = random.randint(int(self.base_size*0.5), int(self.base_size*2.0))
+        if h > w:
+            oh = long_size
+            ow = int(1.0 * w * long_size / h + 0.5)
+            short_size = ow
+        else:
+            ow = long_size
+            oh = int(1.0 * h * long_size / w + 0.5)
+            short_size = oh
+        img = img.resize((ow, oh), Image.BILINEAR)
+        mask = mask.resize((ow, oh), Image.NEAREST)
+        # pad crop
+        if short_size < crop_size:
+            padh = crop_size - oh if oh < crop_size else 0
+            padw = crop_size - ow if ow < crop_size else 0
+            img = ImageOps.expand(img, border=(0, 0, padw, padh), fill=0)
+            mask = ImageOps.expand(mask, border=(0, 0, padw, padh), fill=0)
+        # random crop crop_size
+        w, h = img.size
+        x1 = random.randint(0, w - crop_size)
+        y1 = random.randint(0, h - crop_size)
+        img = img.crop((x1, y1, x1+crop_size, y1+crop_size))
+        mask = mask.crop((x1, y1, x1+crop_size, y1+crop_size))
+        # final transform
+        return img, self._mask_transform(mask)
+
+    def _mask_transform(self, mask):
+        return torch.from_numpy(np.array(mask)).long()
+

datasets/cityscapes.py

@@ -0,0 +1,89 @@
+import os
+import sys
+import random
+import numpy as np
+from tqdm import tqdm, trange
+from PIL import Image, ImageOps, ImageFilter
+
+import torch
+import torch.utils.data as data
+import torchvision.transforms as transform
+
+from datasets.base import BaseDataset
+
+class CitySegmentation(BaseDataset):
+    NUM_CLASS = 19
+    def __init__(self, root, split='val', mode='testval', transform=None, target_transform=None, **kwargs):
+        super(CitySegmentation, self).__init__(
+            root, split, mode, transform, target_transform, **kwargs)
+        self.images, self.mask_paths = get_city_pairs(self.root, self.split)
+        assert (len(self.images) == len(self.mask_paths))
+        if len(self.images) == 0:
+            raise RuntimeError("Found 0 images in subfolders of: \
+                " + self.root + "\n")
+        self._indices = np.array(range(-1, 19))
+        self._classes = np.array([0, 7, 8, 11, 12, 13, 17, 19, 20, 21, 22,
+                                  23, 24, 25, 26, 27, 28, 31, 32, 33])
+        self._key = np.array([-1, -1, -1, -1, -1, -1,
+                              -1, -1,  0,  1, -1, -1, 
+                              2,   3,  4, -1, -1, -1,
+                              5,  -1,  6,  7,  8,  9,
+                              10, 11, 12, 13, 14, 15,
+                              -1, -1, 16, 17, 18])
+        self._mapping = np.array(range(-1, len(self._key)-1)).astype('int32')
+
+    def _class_to_index(self, mask):
+        # assert the values
+        values = np.unique(mask)
+        for i in range(len(values)):
+            assert(values[i] in self._mapping)
+        index = np.digitize(mask.ravel(), self._mapping, right=True)
+        return self._key[index].reshape(mask.shape)
+
+    def __getitem__(self, index):
+        img = Image.open(self.images[index]).convert('RGB')
+        mask = Image.open(self.mask_paths[index])
+        if self.mode == 'testval':
+            img, mask = self._testval_transform(img, mask)
+        elif self.mode == 'val':
+            img, mask = self._val_transform(img, mask)
+        elif self.mode == 'train':
+            img, mask = self._train_transform(img, mask)
+
+        if self.transform is not None:
+            img = self.transform(img)
+        if self.target_transform is not None:
+            mask = self.target_transform(mask)
+        return img, mask
+
+    def _mask_transform(self, mask):
+        target = self._class_to_index(np.array(mask).astype('int32'))
+        return torch.from_numpy(target).long()
+
+    def __len__(self):
+        return len(self.images)
+
+
+def get_city_pairs(folder, split='val'):
+    def get_path_pairs(img_folder, mask_folder):
+        img_paths = []  
+        mask_paths = []  
+        for root, directories, files in os.walk(img_folder):
+            for filename in files:
+                if filename.endswith(".png"):
+                    imgpath = os.path.join(root, filename)
+                    foldername = os.path.basename(os.path.dirname(imgpath))
+                    maskname = filename.replace('leftImg8bit','gtFine_labelIds')
+                    maskpath = os.path.join(mask_folder, foldername, maskname)
+                    if os.path.isfile(imgpath) and os.path.isfile(maskpath):
+                        img_paths.append(imgpath)
+                        mask_paths.append(maskpath)
+                    else:
+                        print('cannot find the mask or image:', imgpath, maskpath)
+        print('Found {} images in the folder {}'.format(len(img_paths), img_folder))
+        return img_paths, mask_paths
+
+    img_folder = os.path.join(folder, 'leftImg8bit/' + split)
+    mask_folder = os.path.join(folder, 'gtFine/'+ split)
+    img_paths, mask_paths = get_path_pairs(img_folder, mask_folder)
+    return img_paths, mask_paths

datasets/utils.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+from torch.utils import data
+import torchvision.transforms as transform
+import torch.nn.functional as F
+from PIL import Image
+import numpy as np
+from collections import defaultdict, deque
+import torch.distributed as dist
+
+def colorize_mask(mask):
+    palette = [128, 64, 128, 244, 35, 232, 70, 70, 70, 102, 102, 156, 190, 153, 153, 153, 153, 153, 250, 170, 30,
+               220, 220, 0, 107, 142, 35, 152, 251, 152, 70, 130, 180, 220, 20, 60, 255, 0, 0, 0, 0, 142, 0, 0, 70,
+               0, 60, 100, 0, 80, 100, 0, 0, 230, 119, 11, 32]
+
+    zero_pad = 256 * 3 - len(palette)
+    for i in range(zero_pad):
+        palette.append(0)
+    new_mask = Image.fromarray(mask.astype(np.uint8)).convert('P')
+    new_mask.putpalette(palette)
+    return new_mask
+
+
+def build_img(args):
+    from PIL import Image
+    img = Image.open(args.input_path)
+    input_transform = transform.Compose([
+        transform.ToTensor(),
+        transform.Normalize([.485, .456, .406], [.229, .224, .225]),
+        transform.Resize((256, 512))])
+    resized_img = input_transform(img)
+    resized_img = resized_img.unsqueeze(0)
+    return resized_img
+
+class ConfusionMatrix(object):
+    def __init__(self, num_classes):
+        self.num_classes = num_classes
+        self.mat = None
+
+    def update(self, a, b):
+        n = self.num_classes
+        if self.mat is None:
+            self.mat = torch.zeros((n, n), dtype=torch.int64, device=a.device)
+        with torch.no_grad():
+            k = (a >= 0) & (a < n)
+            inds = n * a[k].to(torch.int64) + b[k]
+            self.mat += torch.bincount(inds, minlength=n**2).reshape(n, n)
+
+    def reset(self):
+        self.mat.zero_()
+
+    def compute(self):
+        h = self.mat.float()
+        acc_global = torch.diag(h).sum() / h.sum()
+        acc = torch.diag(h) / h.sum(1)
+        iu = torch.diag(h) / (h.sum(1) + h.sum(0) - torch.diag(h))
+        return acc_global, acc, iu
+
+    def reduce_from_all_processes(self):
+        if not torch.distributed.is_available():
+            return
+        if not torch.distributed.is_initialized():
+            return
+        torch.distributed.barrier()
+        torch.distributed.all_reduce(self.mat)
+
+    def __str__(self):
+        acc_global, acc, iu = self.compute()
+        
+        return (
+            'per-class IoU(%): \n {}\n'
+            'mean IoU(%): {:.1f}').format(
+             ['{:.1f}'.format(i) for i in (iu * 100).tolist()],
+             iu.mean().item() * 100)

infer_onnx.py

@@ -0,0 +1,49 @@
+import os
+import sys
+import pathlib
+CURRENT_DIR = pathlib.Path(__file__).parent
+sys.path.append(str(CURRENT_DIR))
+
+import numpy as np
+from tqdm import tqdm
+import torch
+import torch.nn as nn
+from torch.utils import data
+import torchvision.transforms as transform
+import torch.nn.functional as F
+import onnxruntime
+from PIL import Image
+import argparse
+from datasets.utils import colorize_mask, build_img
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='SemanticFPN model')
+    parser.add_argument('--onnx_path', type=str, default='FPN_int.onnx')
+    parser.add_argument('--save_path', type=str, default='./data/demo_results/senmatic_results.png')
+    parser.add_argument('--input_path', type=str, default='data/cityscapes/leftImg8bit/test/bonn/bonn_000000_000019_leftImg8bit.png')
+    parser.add_argument('--ipu', action='store_true', help='use ipu')
+    parser.add_argument('--provider_config', type=str, default=None,
+                    help='provider config path')
+    args = parser.parse_args()
+
+    if args.ipu:
+        providers = ["VitisAIExecutionProvider"]
+        provider_options = [{"config_file": args.provider_config}]
+    else:
+        providers = ['CPUExecutionProvider']
+        provider_options = None
+
+    onnx_path = args.onnx_path
+    input_img = build_img(args)
+    session = onnxruntime.InferenceSession(onnx_path, providers=providers, provider_options=provider_options)
+    ort_input = {session.get_inputs()[0].name: input_img.cpu().numpy()}
+    ort_output = session.run(None, ort_input)[0]
+    if isinstance(ort_output, (tuple, list)):
+        ort_output = ort_output[0]
+
+    output = ort_output[0].transpose(1, 2, 0)
+    seg_pred = np.asarray(np.argmax(output, axis=2), dtype=np.uint8)
+    color_mask = colorize_mask(seg_pred)
+    os.makedirs(os.path.dirname(args.save_path), exist_ok=True)
+    color_mask.save(args.save_path)

requirements.txt

@@ -0,0 +1,9 @@
+torch==1.13.1
+torchvision==0.14.1
+numpy>=1.23.5
+scipy>=1.9
+opencv-python
+pandas
+pillow
+scikit-image
+tqdm

test_onnx.py

@@ -0,0 +1,105 @@
+import os
+import sys
+import pathlib
+CURRENT_DIR = pathlib.Path(__file__).parent
+sys.path.append(str(CURRENT_DIR))
+
+import numpy as np
+from tqdm import tqdm
+import torch
+import torch.nn as nn
+from torch.utils import data
+import torchvision.transforms as transform
+import torch.nn.functional as F
+import onnxruntime
+from PIL import Image
+import argparse
+import datasets.utils as utils
+
+class Configs():
+    def __init__(self):
+        parser = argparse.ArgumentParser(description='PyTorch SemanticFPN model')
+        # dataset
+
+        parser.add_argument('--dataset', type=str, default='citys', help='dataset name (default: citys)')
+        parser.add_argument('--onnx_path', type=str, default='FPN_int.onnx', help='onnx path')
+        parser.add_argument('--num-classes', type=int, default=19,
+                            help='the classes numbers (default: 19 for cityscapes)')
+        parser.add_argument('--test-folder', type=str, default='./data/cityscapes',
+                            help='test dataset folder (default: ./data/cityscapes)')
+
+        parser.add_argument('--base-size', type=int, default=1024, help='the shortest image size')
+        parser.add_argument('--crop-size', type=int, default=256, help='input size for inference')
+        parser.add_argument('--batch-size', type=int, default=1, metavar='N',
+                            help='input batch size for testing (default: 10)')
+        # ipu setting
+        parser.add_argument('--ipu', action='store_true', help='use ipu')
+        parser.add_argument('--provider_config', type=str, default=None, help='provider config path')
+
+        self.parser = parser
+
+    def parse(self):
+        args = self.parser.parse_args()
+        print(args)
+        return args
+
+
+def build_data(args, subset_len=None, sample_method='random'):
+    from datasets import get_segmentation_dataset
+    input_transform = transform.Compose([
+        transform.ToTensor(),
+        transform.Normalize([.485, .456, .406], [.229, .224, .225])])
+
+    data_kwargs = {'transform': input_transform, 'base_size': args.base_size, 'crop_size': args.crop_size}
+
+    testset = get_segmentation_dataset(args.dataset, split='val', mode='testval', root=args.test_folder,
+                                           **data_kwargs)
+    if subset_len:
+        assert subset_len <= len(testset)
+        if sample_method == 'random':
+            testset = torch.utils.data.Subset(testset, random.sample(range(0, len(test_data)), subset_len))
+        else:
+            testset = torch.utils.data.Subset(testset, list(range(subset_len)))
+    # dataloader
+    test_data = data.DataLoader(testset, batch_size=args.batch_size, drop_last=False, shuffle=False)
+    return test_data
+
+
+def eval_miou(data,path="FPN_int.onnx", device='cpu'):
+    confmat = utils.ConfusionMatrix(args.num_classes)
+    tbar = tqdm(data, desc='\r')
+    if args.ipu:
+        providers = ["VitisAIExecutionProvider"]
+        provider_options = [{"config_file": args.provider_config}]
+    else:
+        providers = ['CPUExecutionProvider']
+        provider_options = None
+    session = onnxruntime.InferenceSession(path, providers=providers, provider_options=provider_options)
+
+    for i, (image, target) in enumerate(tbar):
+        image, target = image.to(device), target.to(device)
+        ort_input = {session.get_inputs()[0].name: image.cpu().numpy()}
+        ort_output = session.run(None, ort_input)[0]
+        if isinstance(ort_output, (tuple, list)):
+            ort_output = ort_output[0]
+        ort_output = torch.from_numpy(ort_output).to(device)
+        if ort_output.size()[2:] != target.size()[1:]:
+            ort_output = F.interpolate(ort_output, size=target.size()[1:], mode='bilinear', align_corners=True)
+
+        confmat.update(target.flatten(), ort_output.argmax(1).flatten())
+
+    confmat.reduce_from_all_processes()
+    print('Evaluation Metric: ')
+    print(confmat)
+
+
+def main(args):
+    print('===> Evaluation mIoU: ')
+    test_data = build_data(args)
+    eval_miou(test_data, args.onnx_path)
+
+
+if __name__ == "__main__":
+    args = Configs().parse()
+    main(args)
+