Repository: opendatalab/dsdl-sdk (language: python)

Issue:
Sherry dev



Discussion / hints:

Fix patch:
diff --git a/dsdl/dataset/__init__.py b/dsdl/dataset/__init__.py
index 4b54373..be4913f 100644
--- a/dsdl/dataset/__init__.py
+++ b/dsdl/dataset/__init__.py
@@ -4,6 +4,7 @@
 from .utils.visualizer import ImageVisualizePipeline
 from .utils import Report
 from .check_dataset import CheckDataset
+from .wrapper_dataset import DSDLDataset
 
 __all__ = [
     "Dataset",
@@ -11,5 +12,6 @@
     "CheckDataset",
     "ImageVisualizePipeline",
     "Util",
-    "Report"
+    "Report",
+    "DSDLDataset",
 ]
diff --git a/dsdl/dataset/base_dataset.py b/dsdl/dataset/base_dataset.py
index 2541838..a2df586 100644
--- a/dsdl/dataset/base_dataset.py
+++ b/dsdl/dataset/base_dataset.py
@@ -35,12 +35,14 @@ def __init__(
             location_config: dict,
             pipeline: Optional[Callable[[Dict], Dict]] = None,
             global_info_type: Union[str, StructMetaclass] = None,
-            global_info: Dict[str, Any] = None
+            global_info: Dict[str, Any] = None,
+            lazy_init: bool = False
     ):
         self.location_config = location_config
         self.pipeline = pipeline  # 处理样本的函数
         self._samples = samples  # 样本所在的yaml文件路径
         self._global_info = global_info
+        self.lazy_init = lazy_init
 
         if isinstance(sample_type, str):
             sample_type = Util.extract_sample_type(sample_type)
@@ -93,19 +95,20 @@ def _load_sample(self):
         该函数的作用是将yaml文件中的样本转换为Struct对象，并存储到sample_list列表中
         """
         sample_list = []
-        for sample in self._samples:
-            sample_list.append(self.sample_type(file_reader=self.file_reader, **sample))
+        for i, sample in enumerate(self._samples):
+            struct_sample = self.sample_type(lazy_init=self.lazy_init, file_reader=self.file_reader, **sample)
+            sample_list.append(self.process_sample(i, struct_sample))
         return sample_list
 
-    def process_sample(self, sample):
+    def process_sample(self, i, sample):
         return sample
 
     def __len__(self):
         return len(self.sample_list)
 
     def __getitem__(self, idx):
-        sample = self.sample_list[idx]
-        data = self.process_sample(sample)
+        data = self.sample_list[idx]
+        # data = self.process_sample(sample)
         if self.pipeline is not None:
             data = self.pipeline(data)
         return data
diff --git a/dsdl/dataset/utils/check.py b/dsdl/dataset/utils/check.py
index 15152d1..5cd6264 100644
--- a/dsdl/dataset/utils/check.py
+++ b/dsdl/dataset/utils/check.py
@@ -46,7 +46,7 @@ def check_struct(sample_type: StructMetaclass, sample: dict, file_reader: BaseFi
             _, nested_report = check_struct(struct_type, sample_value, file_reader)
             if not nested_report["normal_flag"]:
                 report["normal_flag"] = False
-                report["warning_flag"] = report["warning_flag"] and nested_report["warning_flag"]
+                report["warning_flag"] = report["warning_flag"] or nested_report["warning_flag"]
                 if "warning_msgs" in nested_report:
                     field_not_matching_msg.extend(nested_report["warning_msgs"])
         elif k in field_mappings:
@@ -60,7 +60,7 @@ def check_struct(sample_type: StructMetaclass, sample: dict, file_reader: BaseFi
                     _, nested_report = check_struct(struct_type, s, file_reader)
                     if not nested_report["normal_flag"]:
                         report["normal_flag"] = False
-                        report["warning_flag"] = report["warning_flag"] and nested_report["warning_flag"]
+                        report["warning_flag"] = report["warning_flag"] or nested_report["warning_flag"]
                         if "warning_msgs" in nested_report:
                             field_not_matching_msg.extend(nested_report["warning_msgs"])
     if not report["normal_flag"]:
@@ -193,6 +193,7 @@ def generate_sample_info(self, file_handler):
 
         res = self.parse_sample_info()
         total_num, normal_num, warn_num, error_num = res["total"], res["normal"], res["warn"], res["error"]
+        assert total_num == normal_num + warn_num + error_num, "Error. Number not match!"
         file_handler.write("samples验证结果如下：" + os.linesep)
         res_str = f"共实例化<font color=blue>{total_num}</font>个样本，" \
                   f"其中<font color=green>{normal_num}</font>个样本实例化正常，" \
diff --git a/dsdl/dataset/utils/visualizer.py b/dsdl/dataset/utils/visualizer.py
index c782d1f..9095f04 100644
--- a/dsdl/dataset/utils/visualizer.py
+++ b/dsdl/dataset/utils/visualizer.py
@@ -13,7 +13,7 @@ def __init__(self, image, palette):
         self.palette = palette
         self.ground_truths = defaultdict(dict)
 
-    def append_ground_truth(self, gt_path, gt_item):
+    def append_ground_truth(self, field_key, gt_path, gt_item):
         """
         {
             ./object/1:{
@@ -28,10 +28,7 @@ def append_ground_truth(self, gt_path, gt_item):
         }
         """
         gt_dir, gt_name = os.path.split(gt_path)
-        if hasattr(gt_item, "field_key"):
-            field_key = gt_item.field_key.lower()
-        else:
-            field_key = gt_item.__class__.__name__.lower()
+
         if field_key not in self.ground_truths[gt_dir]:
             self.ground_truths[gt_dir][field_key] = {gt_name: gt_item}
         else:
@@ -118,7 +115,7 @@ def __init__(self, field_list: List[str], sample: Struct, palette: Optional[Dict
             self.palette = palette
         if "image" not in field_list:  # 因为是imagevisualizer类，所以field_list中必须包含image字段
             raise ValueError("'image' field not found in the field list to be visualized.")
-        self.data_dic = sample.extract_field_info(field_list)
+        self.data_dic = sample.extract_field_info(field_list, verbose=True)
         self.visualize_result = self.group_media_and_ann()
 
     @classmethod
@@ -147,11 +144,11 @@ def group_media_and_ann(self):
         image_dic = data_dic.pop("image")
         image_paths = list(image_dic.keys())
         result_dic = {k_: ImageSample(image_dic[k_], self.palette) for k_ in image_paths}
-        for ann_dic in data_dic.values():
+        for field_key, ann_dic in data_dic.items():
             for ann_path, ann_obj in ann_dic.items():
                 matched_img_paths = self._match(ann_path, image_paths)
                 for matched_img_path in matched_img_paths:
-                    result_dic[matched_img_path].append_ground_truth(ann_path, ann_obj)
+                    result_dic[matched_img_path].append_ground_truth(field_key, ann_path, ann_obj)
         return result_dic
 
     def visualize(self):
diff --git a/dsdl/dataset/wrapper_dataset.py b/dsdl/dataset/wrapper_dataset.py
new file mode 100644
index 0000000..2fdf0f0
--- /dev/null
+++ b/dsdl/dataset/wrapper_dataset.py
@@ -0,0 +1,109 @@
+from .base_dataset import Dataset
+from yaml import load as yaml_load
+from typing import Sequence, Union
+
+try:
+    from yaml import CSafeLoader as YAMLSafeLoader
+except ImportError:
+    from yaml import SafeLoader as YAMLSafeLoader
+import os
+import json
+from ..parser import dsdl_parse
+from .utils.commons import Util
+from ..geometry import CLASSDOMAIN
+
+
+class DSDLDataset(Dataset):
+    YAML_VALID_SUFFIX = ('.yaml', '.YAML')
+    JSON_VALID_SUFFIX = ('.json', '.JSON')
+    VALID_SUFFIX = YAML_VALID_SUFFIX + JSON_VALID_SUFFIX
+
+    def __init__(self, dsdl_yaml, location_config, import_dir='', lazy_init=True):
+        self._dsdl_yaml = dsdl_yaml
+        self._location_config = location_config
+        self._import_dir = import_dir
+
+        self._yaml_info = self.extract_info_from_yml()
+        dsdl_py, sample_type, samples, global_info_type, global_info = self._yaml_info["dsdl_py"], self._yaml_info[
+            "sample_type"], self._yaml_info["samples"], self._yaml_info["global_info_type"], self._yaml_info[
+                                                                           "global_info"]
+        exec(dsdl_py, {})
+        all_class_dom = Util.extract_class_dom(sample_type)
+        self.class_dom = None
+        this_class_dom = None
+        for _arg in all_class_dom:
+            for _dom_ind, class_dom in enumerate(all_class_dom[_arg]):
+                this_class_dom = CLASSDOMAIN.get(class_dom)
+                all_class_dom[_arg][_dom_ind] = this_class_dom
+        self.class_dom = this_class_dom
+        self.meta = self._yaml_info["meta"]
+        self.version = self._yaml_info["version"]
+
+        super().__init__(samples, sample_type, location_config, None, global_info_type, global_info, lazy_init)
+
+    def extract_info_from_yml(self):
+        dsdl_yaml = self._dsdl_yaml
+        with open(dsdl_yaml, "r") as f:
+            dsdl_all_info = yaml_load(f, Loader=YAMLSafeLoader)
+        dsdl_info, dsdl_meta, dsdl_version = dsdl_all_info['data'], dsdl_all_info["meta"], dsdl_all_info[
+            "$dsdl-version"]
+        sample_type = dsdl_info['sample-type']
+        global_info_type = dsdl_info.get("global-info-type", None)
+        global_info = None
+        if "sample-path" not in dsdl_info or dsdl_info["sample-path"] in ("local", "$local"):
+            assert "samples" in dsdl_info, f"Key 'samples' is required in {dsdl_yaml}."
+            samples = dsdl_info['samples']
+        else:
+            sample_path = dsdl_info["sample-path"]
+            samples = self.load_samples(dsdl_yaml, sample_path)
+        if global_info_type is not None:
+            if "global-info-path" not in dsdl_info:
+                assert "global-info" in dsdl_info, f"Key 'global-info' is required in {dsdl_yaml}."
+                global_info = dsdl_info["global_info"]
+            else:
+                global_info_path = dsdl_info["global-info-path"]
+                global_info = self.load_samples(dsdl_yaml, global_info_path, "global-info")[0]
+
+        dsdl_py = dsdl_parse(dsdl_yaml, dsdl_library_path=self._import_dir)
+
+        res = {
+            "sample_type": sample_type,
+            "global_info_type": global_info_type,
+            "samples": samples,
+            "global_info": global_info,
+            "dsdl_py": dsdl_py,
+            "version": dsdl_version,
+            "meta": dsdl_meta
+        }
+        return res
+
+    @classmethod
+    def load_samples(cls, dsdl_path: str, path: Union[str, Sequence[str]], extract_key="samples"):
+        samples = []
+        paths = []
+        dsdl_dir = os.path.split(dsdl_path)[0]
+        if isinstance(path, str):
+            path = os.path.join(dsdl_dir, path)
+            if os.path.isdir(path):
+                paths = [os.path.join(path, _) for _ in os.listdir(path) if _.endswith(cls.VALID_SUFFIX)]
+            elif os.path.isfile(path):
+                if path.endswith(cls.VALID_SUFFIX):
+                    paths = [path]
+        elif isinstance(path, (list, tuple)):
+            paths = [os.path.join(dsdl_dir, _) for _ in path if os.path.isfile(_) and _.endswith(cls.VALID_SUFFIX)]
+        for p in paths:
+            if p.endswith(cls.YAML_VALID_SUFFIX):
+                with open(p, "r") as f:
+                    data = yaml_load(f, YAMLSafeLoader)[extract_key]
+                if isinstance(data, list):
+                    samples.extend(data)
+                else:
+                    samples.append(data)
+            else:
+                with open(p, "r") as f:
+                    data = json.load(f)[extract_key]
+                if isinstance(data, list):
+                    samples.extend(data)
+                else:
+                    samples.append(data)
+        return samples
diff --git a/dsdl/geometry/__init__.py b/dsdl/geometry/__init__.py
index b2782db..4747790 100644
--- a/dsdl/geometry/__init__.py
+++ b/dsdl/geometry/__init__.py
@@ -11,7 +11,7 @@
 from .text import Text
 from .rotate_box import RBBox
 from .shape import Shape, ImageShape
-from .instanceid import InstanceID
+from .uniqueid import UniqueID
 
 __all__ = [
     "BBox",
@@ -33,5 +33,5 @@
     "RBBox",
     "Shape",
     "ImageShape",
-    "InstanceID",
+    "UniqueID",
 ]
diff --git a/dsdl/geometry/attrbutes.py b/dsdl/geometry/attrbutes.py
index bb5bb30..e7c16c1 100644
--- a/dsdl/geometry/attrbutes.py
+++ b/dsdl/geometry/attrbutes.py
@@ -1,5 +1,6 @@
 from .base_geometry import BaseGeometry
 
+
 class Attributes(BaseGeometry):
     def __init__(self, **kwargs):
         self.container = {}
diff --git a/dsdl/geometry/base_geometry.py b/dsdl/geometry/base_geometry.py
index d09ad3b..9f3630a 100644
--- a/dsdl/geometry/base_geometry.py
+++ b/dsdl/geometry/base_geometry.py
@@ -9,3 +9,15 @@ def __repr__(self):
     @property
     def field_key(self):
         return self.__class__.__name__
+
+
+class FontMixin:
+    FONT = None
+
+    @classmethod
+    def set_font(cls, font):
+        cls.FONT = font
+
+    @property
+    def font(self):
+        return self.FONT
diff --git a/dsdl/geometry/class_domain.py b/dsdl/geometry/class_domain.py
index 5b7ef0b..a3b08f7 100644
--- a/dsdl/geometry/class_domain.py
+++ b/dsdl/geometry/class_domain.py
@@ -17,20 +17,22 @@ def __new__(mcs, name, bases, attributes):
         classes = attributes.pop('Classes', [])
         classes = [_ for _ in classes if isinstance(_, Label)]
         mapping = {}
-        for label in classes:
+        cat2ind = {}
+        for ind, label in enumerate(classes):
             label.set_domain(name)
             mapping[label.name] = label
+            cat2ind[label.name] = ind + 1
             LABEL.registry(label)
 
+        attributes["__cat2ind_mapping__"] = cat2ind
         attributes["__mapping__"] = mapping
         attributes["__list__"] = classes
         attr_dic = {attr_k: CLASSDOMAIN_ATTRIBUTES[attr_k](attributes.pop(attr_k)) for attr_k in
-                                        CLASSDOMAIN_ATTRIBUTES if attr_k in attributes}
+                    CLASSDOMAIN_ATTRIBUTES if attr_k in attributes}
         for attr_k in attr_dic:
             attr_dic[attr_k].set_domain(name)
         attributes["__attributes__"] = attr_dic
 
-
         new_cls = super_new(mcs, name, bases, attributes)
         CLASSDOMAIN.register(name, new_cls)
         return new_cls
@@ -59,6 +61,10 @@ def get_label_names(cls):
         labels = cls.get_labels()
         return [_.name for _ in labels]
 
+    @classmethod
+    def get_cat2ind_mapping(cls):
+        return getattr(cls, '__cat2ind_mapping__')
+
     @classmethod
     def get_label(cls, name):
         if isinstance(name, str):
@@ -80,3 +86,10 @@ def get_label(cls, name):
     def get_attribute(cls, attr_name):
         attr_dic = getattr(cls, "__attributes__")
         return attr_dic.get(attr_name, None)
+
+
+class _LabelMapDefaultDomain(ClassDomain):
+    Classes = [
+        Label("background"),
+        Label("object"),
+    ]
diff --git a/dsdl/geometry/label.py b/dsdl/geometry/label.py
index deef7f1..0112b68 100644
--- a/dsdl/geometry/label.py
+++ b/dsdl/geometry/label.py
@@ -1,10 +1,12 @@
+import os
 import numpy as np
 from PIL import ImageFont, ImageDraw
 from .registry import CLASSDOMAIN
-from .base_geometry import BaseGeometry
+from .base_geometry import BaseGeometry, FontMixin
 
 
-class Label(BaseGeometry):
+class Label(BaseGeometry, FontMixin):
+
     def __init__(self, name, supercategories=(), domain_name=None):
         self._name = name
         self._supercategories = [_ for _ in supercategories if isinstance(_, Label)]
@@ -55,9 +57,7 @@ def class_domain(self):
         return CLASSDOMAIN.get(self.domain_name)
 
     def index_in_domain(self):
-        all_label_names = self.class_domain.get_label_names()
-        ind = all_label_names.index(self.category_name) + 1
-        return ind
+        return self.class_domain.get_cat2ind_mapping()[self.category_name]
 
     def __eq__(self, other):
         if self.domain_name != other.domain_name:
@@ -77,8 +77,9 @@ def visualize(self, image, palette, **kwargs):
         if self.category_name not in palette:
             palette[self.category_name] = tuple(np.random.randint(0, 255, size=[3]))
         color = palette[self.category_name]
-        font = ImageFont.load_default()
-        label_size = draw_obj.textsize(self.category_name, font)
+        if self.font is None:
+            self.set_font(ImageFont.truetype(os.path.join(os.path.dirname(__file__), "source", "Arial_Font.ttf")))
+        label_size = draw_obj.textsize(self.category_name, self.font)
         if "bbox" in kwargs:
             coords = np.array([[item.xyxy[0], item.xyxy[1] + 0.2 * label_size[1]] for item in kwargs["bbox"].values()])
         elif "polygon" in kwargs:
@@ -88,7 +89,7 @@ def visualize(self, image, palette, **kwargs):
             coords = np.array([[0, 0.2 * label_size[1]]])
         for coord in coords:
             draw_obj.rectangle([tuple(coord), tuple(coord + label_size)], fill=(*color, 255))
-            draw_obj.text(tuple(coord), self.category_name, fill=(255, 255, 255, 255), font=font)
+            draw_obj.text(tuple(coord), self.category_name, fill=(255, 255, 255, 255), font=self.font)
         del draw_obj
         return image
 
@@ -100,7 +101,7 @@ def field_key(self):
         return "Label"
 
 
-class LabelList(BaseGeometry):
+class LabelList(BaseGeometry, FontMixin):
 
     def __init__(self, label_list):
         self._label_list = list(label_list)
@@ -123,7 +124,8 @@ def class_domains(self):
 
     def visualize(self, image, palette, **kwargs):
         draw_obj = ImageDraw.Draw(image)
-        font = ImageFont.load_default()
+        if self.font is None:
+            self.set_font(ImageFont.truetype(os.path.join(os.path.dirname(__file__), "source", "Arial_Font.ttf")))
         y_offset = np.zeros((1, 2))
         for label_obj in self.label_list:
 
@@ -133,7 +135,7 @@ def visualize(self, image, palette, **kwargs):
                 palette[category_name] = tuple(np.random.randint(0, 255, size=[3]))
             color = palette[category_name]
 
-            label_size = draw_obj.textsize(category_name, font)
+            label_size = draw_obj.textsize(category_name, self.font)
             if "bbox" in kwargs:
                 # coords.shape = [num_box, 2]
                 coords = y_offset + np.array(
@@ -155,7 +157,7 @@ def visualize(self, image, palette, **kwargs):
             y_offset += np.array([[0., 1.2 * label_size[1]]])
             for coord in coords:
                 draw_obj.rectangle([tuple(coord), tuple(coord + label_size)], fill=(*color, 255))
-                draw_obj.text(tuple(coord), category_name, fill=(255, 255, 255, 255), font=font)
+                draw_obj.text(tuple(coord), category_name, fill=(255, 255, 255, 255), font=self.font)
 
         del draw_obj
         return image
diff --git a/dsdl/geometry/rotate_box.py b/dsdl/geometry/rotate_box.py
index 1f35a05..16aba21 100644
--- a/dsdl/geometry/rotate_box.py
+++ b/dsdl/geometry/rotate_box.py
@@ -7,7 +7,8 @@
 
 class RBBox(BaseGeometry):
     def __init__(self, value, mode):
-        if mode == "xywhr":
+        assert mode in ("xywht", "xyxy")
+        if mode == "xywht":
             self._polygon = None
             self._rbbox = value
         else:
diff --git a/dsdl/geometry/segmap.py b/dsdl/geometry/segmap.py
index 081e6bb..bd3cc08 100644
--- a/dsdl/geometry/segmap.py
+++ b/dsdl/geometry/segmap.py
@@ -6,6 +6,7 @@
 import io
 from .label import LabelList
 from .box import BBox
+from .class_domain import _LabelMapDefaultDomain
 
 
 class SegmentationMap(BaseGeometry):
@@ -17,6 +18,8 @@ def __init__(self, location, file_reader, dom):
         self._loc = location
         self._reader = file_reader
         self._dom = dom
+        if self._dom is None:
+            self._dom = _LabelMapDefaultDomain
 
     @property
     def class_domain(self):
@@ -52,6 +55,8 @@ def visualize(self, image, palette, **kwargs):
         seg = self.to_array()
         color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
         category_ids = np.unique(seg)
+        if self.class_domain.__name__ == "_LabelMapDefaultDomain":
+            category_ids = category_ids + 1
         label_lst = []
         for category_id in category_ids:
             if int(category_id) > len(self._dom) or int(category_id) < 1:
diff --git a/dsdl/geometry/source/Arial_Font.ttf b/dsdl/geometry/source/Arial_Font.ttf
new file mode 100644
index 0000000..51a18bc
Binary files /dev/null and b/dsdl/geometry/source/Arial_Font.ttf differ
diff --git a/dsdl/geometry/text.py b/dsdl/geometry/text.py
index cf04de6..c2b01c3 100644
--- a/dsdl/geometry/text.py
+++ b/dsdl/geometry/text.py
@@ -1,9 +1,10 @@
-from .base_geometry import BaseGeometry
+from .base_geometry import BaseGeometry, FontMixin
 from PIL import ImageDraw, ImageFont
 import numpy as np
+import os
 
 
-class Text(BaseGeometry):
+class Text(BaseGeometry, FontMixin):
     def __init__(self, text):
         self._text = text
 
@@ -18,8 +19,9 @@ def text(self):
     def visualize(self, image, palette, **kwargs):
         draw_obj = ImageDraw.Draw(image)
         text_color = (0, 255, 0)  # green
-        font = ImageFont.load_default()
-        label_size = draw_obj.textsize(self.value, font)
+        if self.font is None:
+            self.set_font(ImageFont.truetype(os.path.join(os.path.dirname(__file__), "source", "Arial_Font.ttf")))
+        label_size = draw_obj.textsize(self.value, self.font)
         if "bbox" in kwargs:
             coords = np.array([[item.xyxy[0], item.xyxy[3] - 1.2 * label_size[1]] for item in kwargs["bbox"].values()])
         elif "polygon" in kwargs:
@@ -31,10 +33,10 @@ def visualize(self, image, palette, **kwargs):
                 [[item.point_for_draw("lb")[0], item.point_for_draw("lb")[1] - 1.2 * label_size[1]] for item in
                  kwargs["rotatedbbox"].values()])
         else:
-            coords = np.array([[0, image.size[0] - 1.2 * label_size[1]]])
+            coords = np.array([[0, image.size[1] - 1.2 * label_size[1]]])
         for coord in coords:
             draw_obj.rectangle([tuple(coord), tuple(coord + label_size)], fill=(*text_color, 255))
-            draw_obj.text(tuple(coord), self.value, fill=(255, 255, 255, 255), font=font)
+            draw_obj.text(tuple(coord), self.value, fill=(255, 255, 255, 255), font=self.font)
         del draw_obj
         return image
 
diff --git a/dsdl/geometry/instanceid.py b/dsdl/geometry/uniqueid.py
similarity index 56%
rename from dsdl/geometry/instanceid.py
rename to dsdl/geometry/uniqueid.py
index ec54757..adf571d 100644
--- a/dsdl/geometry/instanceid.py
+++ b/dsdl/geometry/uniqueid.py
@@ -1,10 +1,11 @@
 from .base_geometry import BaseGeometry
 
 
-class InstanceID(BaseGeometry):
+class UniqueID(BaseGeometry):
 
-    def __init__(self, value):
+    def __init__(self, value, field_key):
         self._value = value
+        self._field_key = field_key
 
     @property
     def value(self):
@@ -14,8 +15,8 @@ def __eq__(self, other):
         return self.value == other.value
 
     def __repr__(self):
-        return f"instance {self.value}"
+        return f"ID type: {self._field_key} ID value: {self.value}"
 
     @property
     def field_key(self):
-        return "InstanceID"
+        return self._field_key
diff --git a/dsdl/objectio/ceph.py b/dsdl/objectio/ceph.py
index 2e55b27..551266b 100644
--- a/dsdl/objectio/ceph.py
+++ b/dsdl/objectio/ceph.py
@@ -1,6 +1,15 @@
 from .base import BaseFileReader
 from contextlib import contextmanager
 import os
+import re
+
+
+class BytesWrapper:
+    def __init__(self, value):
+        self.value = value
+
+    def read(self):
+        return self.value
 
 
 class CephFileReader(BaseFileReader):
@@ -19,20 +28,20 @@ def load(self, filepath):
         filepath = os.path.join(self.working_dir, filepath)
         value = self._client.Get(filepath)
         try:
-            yield memoryview(value)
+            yield BytesWrapper(value)
         finally:
             pass
 
 
 class PetrelFileReader(BaseFileReader):
-    def __init__(self, working_dir, enable_mc=True):
+    def __init__(self, working_dir, conf_path=None):
         super().__init__(working_dir)
         try:
             from petrel_client import client
         except ImportError:
             raise ImportError('Please install petrel_client to enable '
                               'PetrelBackend.')
-        self._client = client.Client(enable_mc=enable_mc)
+        self._client = client.Client(conf_path=conf_path)
 
     def _format_path(self, filepath: str) -> str:
         """Convert a ``filepath`` to standard format of petrel oss.
@@ -47,10 +56,12 @@ def _format_path(self, filepath: str) -> str:
         """
         return re.sub(r'\\+', '/', filepath)
 
+    @contextmanager
     def load(self, filepath):
+        filepath = os.path.join(self.working_dir, filepath)
         filepath = self._format_path(filepath)
         value = self._client.Get(filepath)
         try:
-            yield memoryview(value)
+            yield BytesWrapper(value)
         finally:
             pass
diff --git a/dsdl/parser/parse_class.py b/dsdl/parser/parse_class.py
index 3b5bc8e..ea1493e 100644
--- a/dsdl/parser/parse_class.py
+++ b/dsdl/parser/parse_class.py
@@ -40,27 +40,27 @@ def _parse_skeleton(self, skeleton: List) -> Optional[List[List[int]]]:
                                 isinstance(ele, int) and len(item) == 2 for ele in item
                         ):
                             raise DefineSyntaxError(
-                                f"Error in skeleton of {self.class_name}: skeleton must be list of list of int."
+                                f"Error in skeleton of `{self.class_name}`: skeleton must be list of list of int."
                             )
                     return skeleton
                 else:
                     raise DefineSyntaxError(
-                        f"Error in skeleton of {self.class_name}: skeleton must be list of list of int."
+                        f"Error in skeleton of `{self.class_name}`: skeleton must be list of list of int."
                     )
             else:
                 raise DefineSyntaxError(
-                    f"Error in skeleton of {self.class_name}: skeleton must be list of list of int."
+                    f"Error in skeleton of `{self.class_name}`: skeleton must be list of list of int."
                 )
 
     def _check_label_name(self, label_name: str) -> str:
         if "." in label_name:
             check_flag = re.search(r"^[\w.]+$", label_name)
             if label_name.startswith("."):
-                raise DefineSyntaxError(f"`{label_name}` is not allowed. Label in class-dom can't starts with dot `.`")
+                raise DefineSyntaxError(f"`{label_name}` is not allowed. Label in class-dom can't start with dot `.`")
             if not check_flag:
                 warn_msg = (
-                    f"`{label_name}` is not recommended."
-                    f" we recommend use alphanumeric letters (a-z, A-Z and 0-9), and underscores (_) "
+                    f"`{label_name}` is not recommended. "
+                    f"We recommend using alphanumeric letters (a-z, A-Z and 0-9), and underscores (_) "
                     f"for label in class-dom (with hierarchical structure)."
                 )
                 warnings.warn(
diff --git a/dsdl/parser/parse_field.py b/dsdl/parser/parse_field.py
index c029b35..7e29832 100644
--- a/dsdl/parser/parse_field.py
+++ b/dsdl/parser/parse_field.py
@@ -20,6 +20,7 @@ def __init__(self, struct_name: Set[str]):
         self.TYPES_LIST = TYPES_LIST
         self.TYPES_IMAGE_SHAPE = TYPES_IMAGE_SHAPE
         self.TYPES_ROTATED_BBOX = TYPES_ROTATED_BBOX
+        self.TYPES_UNIQUE_ID = TYPES_UNIQUE_ID
         self.TYPES_ALL = TYPES_ALL
         self.is_attr = set()
         self.optional = set()
@@ -52,17 +53,25 @@ def sanitize_ordered(val: str) -> str:
                     return "False"
             else:
                 raise DefineSyntaxError(
-                    f"invalid value {val} in ordered of {field_type}: {','.join(param_list)}."
+                    f"invalid value {val} in `ordered` of `{field_type}` field: check the '{','.join(param_list)}'."
                 )
 
         ele_type, ordered = None, None
         if len(param_list) == 2:
-            ele_type = param_list[0].strip()
-            ordered = param_list[1].strip()
+            if param_list[0].strip().startswith("etype"):
+                ele_type = param_list[0].strip()
+                ordered = param_list[1].strip()
+            elif param_list[1].strip().startswith("etype"):
+                ordered = param_list[0].strip()
+                ele_type = param_list[1].strip()
+            else:
+                raise DefineSyntaxError(
+                    f"Invalid parameters {param_list} in List field."
+                )
         elif len(param_list) == 1:
             ele_type = param_list[0].strip()
         else:
-            raise DefineSyntaxError(f"invalid parameters {param_list} in List.")
+            raise DefineSyntaxError(f"Invalid parameters {param_list} in List field.")
 
         if ele_type.startswith("etype"):
             # if ele_type like "etype=LocalObjectEntry[cdom=$objectdom]", split by "="
@@ -71,12 +80,12 @@ def sanitize_ordered(val: str) -> str:
             if len(ele_type) == 2:
                 if ele_type[0].strip() != "etype":
                     raise DefineSyntaxError(
-                        f"List types must contains parameters `etype`."
+                        f"List field must contain parameter `etype`."
                     )
                 ele_type = ele_type[1].strip()
             else:
                 raise DefineSyntaxError(
-                    f"invalid parameters {', '.join(param_list)} in {field_type}."
+                    f"Invalid parameters {', '.join(param_list)} in {field_type} field."
                 )
 
         res = field_type + "Field("
@@ -84,7 +93,7 @@ def sanitize_ordered(val: str) -> str:
             ele_type = sanitize_etype(ele_type)
             res += "ele_type=" + ele_type
         else:
-            raise DefineSyntaxError(f"List types must contains parameters `etype`.")
+            raise DefineSyntaxError(f"List field must contain parameter `etype`.")
         if ordered:
             ordered = ordered.split("=")[-1]
             ordered = sanitize_ordered(ordered)
@@ -124,16 +133,16 @@ def sanitize_fmt(val: str) -> str:
                     field_var = parts[0]
                 else:
                     raise DefineSyntaxError(
-                        f"invalid parameters {param} in {field_type}."
+                        f"invalid parameters {param} in {field_type} field."
                     )
 
                 if field_para != "fmt":
                     raise DefineSyntaxError(
-                        f"invalid parameters {field_para} in {field_type}."
+                        f"invalid parameters {field_para} in {field_type} field."
                     )
 
                 if field_para in param_dict:
-                    raise ValueError(f"duplicated param {param} in {field_type}.")
+                    raise ValueError(f"duplicated param {param} in {field_type} field.")
                 else:
                     param_dict[field_para] = sanitize_fmt(field_var)
 
@@ -160,7 +169,7 @@ def sanitize_mode(val: str) -> str:
             if val in temp:
                 return f'"{val}"'
             raise DefineSyntaxError(
-                f"""invalid parameters value "{val}" of `mode` in `{field_type}`, """
+                f"""invalid parameters value "{val}" of `mode` in `{field_type}` field, """
                 f"""`mode` value must be one of ["{'", "'.join(temp)}"]."""
             )
 
@@ -180,17 +189,17 @@ def sanitize_mode(val: str) -> str:
                     field_var = parts[0]
                 else:
                     raise DefineSyntaxError(
-                        f"invalid parameters {param} in {field_type}."
+                        f"invalid parameters {param} in {field_type} field."
                     )
 
                 if field_para in param_dict:
-                    raise ValueError(f"duplicated param {param} in {field_type}.")
+                    raise ValueError(f"duplicated param {param} in {field_type} field.")
                 else:
                     if field_para == "mode":
                         param_dict[field_para] = sanitize_mode(field_var)
                     else:
                         raise DefineSyntaxError(
-                            f"invalid parameters '{field_para}' in {field_type}."
+                            f"invalid parameters '{field_para}' in {field_type} field."
                         )
         return (
             field_type
@@ -202,7 +211,7 @@ def sanitize_mode(val: str) -> str:
     @staticmethod
     def parse_rotated_bbox_field(field_type: str, param_list: List[str]) -> str:
         """
-        解析处理ImageShape类型的field
+        解析处理RotatedBBox类型的field
         """
 
         def sanitize_mode(val: str) -> str:
@@ -215,7 +224,7 @@ def sanitize_mode(val: str) -> str:
             if val in temp:
                 return f'"{val}"'
             raise DefineSyntaxError(
-                f"""invalid parameters value "{val}" of `mode` in `{field_type}`, """
+                f"""invalid parameters value "{val}" of `mode` in `{field_type}` field, """
                 f"""`mode` value must be one of ["{'", "'.join(temp)}"]."""
             )
 
@@ -229,8 +238,8 @@ def sanitize_measure(val: str) -> str:
             if val in temp:
                 return f'"{val}"'
             raise DefineSyntaxError(
-                f"""invalid parameters value "{val}" of `measure` in `{field_type}`, """
-                f"""`mode` value must be one of ["{'", "'.join(temp)}"]."""
+                f"""invalid parameters value "{val}" of `measure` in `{field_type}` field, """
+                f"""`measure` value must be one of ["{'", "'.join(temp)}"]."""
             )
 
         param_dict = dict()
@@ -250,11 +259,11 @@ def sanitize_measure(val: str) -> str:
                     field_var = parts[0]
                 else:
                     raise DefineSyntaxError(
-                        f"invalid parameters {param} in {field_type}."
+                        f"invalid parameters {param} in {field_type} field."
                     )
 
                 if field_para in param_dict:
-                    raise ValueError(f"duplicated param {param} in {field_type}.")
+                    raise ValueError(f"duplicated param {param} in {field_type} field.")
                 else:
                     if field_para == "mode":
                         param_dict[field_para] = sanitize_mode(field_var)
@@ -262,7 +271,63 @@ def sanitize_measure(val: str) -> str:
                         param_dict[field_para] = sanitize_measure(field_var)
                     else:
                         raise DefineSyntaxError(
-                            f"invalid parameters '{field_para}' in {field_type}."
+                            f"invalid parameters '{field_para}' in {field_type} field."
+                        )
+        return (
+            field_type
+            + "Field("
+            + ", ".join([f"{k}={v}" for k, v in param_dict.items()])
+            + ")"
+        )
+
+    @staticmethod
+    def parse_unique_id_field(field_type: str, param_list: List[str]) -> str:
+        """
+        解析处理ImageShape类型的field
+        """
+
+        def sanitize_id_type(val: str) -> str:
+            """
+            UniqueID中对id_type部分的校验，是用来严格限制特定格式或者字符。
+            防止yaml里有一些异常代码注入到生成的Python 代码里被执行起来。
+            """
+            try:
+                val = str(val)
+                val = val.strip("\"'")
+            except NameError:
+                raise DefineSyntaxError(
+                    f"""invalid parameters value "{val}" of `id_type` in `{field_type}` field, """
+                    f"""`id_type` value must be a string."""
+                )
+            return f'"{val}"'
+
+        param_dict = dict()
+        if not param_list:
+            param_dict["id_type"] = sanitize_id_type("None")
+        else:
+            for param in param_list:
+                parts = param.split("=")
+                parts = [i.strip() for i in parts]
+                # 需要考虑参数省略的情况，因为mode经常省略
+                if len(parts) == 2:
+                    field_para = parts[0]
+                    field_var = parts[1]
+                elif len(parts) == 1:
+                    field_para = "id_type"
+                    field_var = parts[0]
+                else:
+                    raise DefineSyntaxError(
+                        f"invalid parameters {param} in {field_type} field."
+                    )
+
+                if field_para in param_dict:
+                    raise ValueError(f"duplicated param {param} in {field_type} field.")
+                else:
+                    if field_para == "id_type":
+                        param_dict[field_para] = sanitize_id_type(field_var)
+                    else:
+                        raise DefineSyntaxError(
+                            f"invalid parameters '{field_para}' in {field_type} field."
                         )
         return (
             field_type
@@ -289,16 +354,16 @@ def sanitize_dom(val: str) -> str:
                     p = p.strip()
                     if not p.isidentifier():
                         raise DefineSyntaxError(
-                            f"invalid dom: {val} for {p} in {val} is illegal."
+                            f"invalid `dom` in List field: `{val}`, for `{p}` in `{val}` is illegal."
                         )
             else:
                 if not val.isidentifier():
-                    raise DefineSyntaxError(f"invalid dom: {val}")
+                    raise DefineSyntaxError(f"invalid dom: `{val}` in List field")
             return val
 
         param_dict = dict()
         if not param_list:
-            raise DefineSyntaxError(f"{field_type} must contains parameter `dom`")
+            raise DefineSyntaxError(f"{field_type} field must contain parameter `dom`")
         for param in param_list:
             parts = param.split("=")
             parts = [i.strip() for i in parts]
@@ -311,15 +376,15 @@ def sanitize_dom(val: str) -> str:
                 field_var = parts[0]
             else:
                 raise DefineSyntaxError(
-                    f"invalid parameters {param_list} in {field_type}."
+                    f"invalid parameters {param_list} in {field_type} field."
                 )
             if field_para != "dom":
                 raise DefineSyntaxError(
-                    f"invalid parameters {field_para} in {field_type}."
+                    f"invalid parameters {field_para} in {field_type} field."
                 )
 
             if field_para in param_dict:
-                raise ValueError(f"duplicated param {param} in {field_type}.")
+                raise ValueError(f"duplicated param {param} in {field_type} field.")
             else:
                 param_dict[field_para] = sanitize_dom(field_var)
 
@@ -345,13 +410,17 @@ def parse_field(
             # 不带参数的不需要校验，直接可以转化为python代码（注意区分yaml中的模型部分到ORM的校验和通过ORM读入数据的校验，
             # 后者在dataset.py中定义，你脑子里想的Int也需要校验是数据的校验。）
             if params_list:
-                raise DefineSyntaxError(f"{field_type} should not contains parameters.")
+                raise DefineSyntaxError(
+                    f"{field_type} should not contain parameters besides [1.optional 2. is_attr]."
+                )
             field_type = field_type + "Field()"
         elif field_type in self.TYPES_LIST:
             # 带参数的List类型的字段的校验
             # （因为List类型比较特殊，里面可以包含List，Label等各种其他字段，涉及递归，所以单独拿出来）
             if not params_list:
-                raise DefineSyntaxError(f"{field_type} must contains parameters.")
+                raise DefineSyntaxError(
+                    f"{field_type} must contain parameters in addition to [1.optional 2. is_attr]."
+                )
             field_type = self.parse_list_filed(field_type, params_list)
         elif field_type in self.TYPES_TIME:
             # 带参数的Date, Time类型的字段的校验
@@ -365,10 +434,13 @@ def parse_field(
         elif field_type in self.TYPES_ROTATED_BBOX:
             # 带参数的RotatedBBox类型的字段的校验
             field_type = self.parse_rotated_bbox_field(field_type, params_list)
+        elif field_type in self.TYPES_UNIQUE_ID:
+            # 带参数的RotatedBBox类型的字段的校验
+            field_type = self.parse_unique_id_field(field_type, params_list)
         else:
             # Struct类型的字段的校验
             if field_type not in self.struct:
-                raise DefineTypeError(f"No type {field_type} in DSDL.")
+                raise DefineTypeError(f"No type `{field_type}` in DSDL.")
             field_type = field_type + "()"
         if attr_flag:
             field_type = add_key_value_2_struct_field(field_type, "is_attr", True)
@@ -383,7 +455,7 @@ def pre_parse_struct_field(
         raw_field_type = raw_field_type.strip()
         fixed_params = re.findall(r"\[(.*)\]", raw_field_type)
         if len(fixed_params) >= 2:
-            raise DefineSyntaxError(f"Error in definition of {raw_field_type} in DSDL.")
+            raise DefineSyntaxError(f"Error in field with value of `{raw_field_type}`.")
         elif len(fixed_params) == 0:
             field_type = self.parse_field(field_type=raw_field_type)
         else:
@@ -396,16 +468,28 @@ def pre_parse_struct_field(
             other_filed = set()
             attr_flag, optional_flag = False, False
             for k_v in k_v_list:
-                if k_v.replace(" ", "") == "is_attr=True":
-                    if field_name:
-                        self.is_attr.add(field_name)
-                    else:
-                        attr_flag = True
-                elif k_v.replace(" ", "") == "optional=True":
-                    if field_name:
-                        self.optional.add(field_name)
+                k_v_temp = k_v.replace(" ", "")
+                if not check_is_bracket_pair(k_v_temp):
+                    raise DefineSyntaxError(
+                        f"Error in field with value of `{raw_field_type}`. Check the `{k_v}` part."
+                    )
+                if k_v_temp.startswith(("is_attr=", "optional=")):
+                    if k_v_temp == "is_attr=True":
+                        if field_name:
+                            self.is_attr.add(field_name)
+                        else:
+                            attr_flag = True
+                    elif k_v_temp == "optional=True":
+                        if field_name:
+                            self.optional.add(field_name)
+                        else:
+                            optional_flag = True
+                    elif k_v_temp in ["optional=False", "is_attr=False"]:
+                        pass
                     else:
-                        optional_flag = True
+                        raise DefineSyntaxError(
+                            f"Error in field with value of {raw_field_type}: check format of `optional` and `is_attr`."
+                        )
                 else:
                     other_filed.add(k_v)
             field_type = self.parse_field(
diff --git a/dsdl/parser/parse_params.py b/dsdl/parser/parse_params.py
index fd579d1..3c2180d 100644
--- a/dsdl/parser/parse_params.py
+++ b/dsdl/parser/parse_params.py
@@ -20,15 +20,18 @@ def __post_init__(self):
         按照规定struct和class dom的名字不能是白皮书中已经包含的类型名，如List这些内定的名字
         """
         if self.struct_name in TYPES_ALL:
-            raise ValidationError(f"{self.struct_name} is dsdl build-in value name, please rename it. "
-                                  f"Build-in value names are: {','.join(TYPES_ALL)}")
-        if self.struct_name in [i + 'Field' for i in TYPES_ALL]:
-            raise ValidationError(f"{self.struct_name} is dsdl build-in value name, please rename it. "
-                                  f"Build-in value names are: {', '.join(TYPES_ALL)}")
+            raise ValidationError(
+                f"{self.struct_name} is dsdl build-in value name, please rename it. "
+                f"Build-in value names are: {','.join(TYPES_ALL)}"
+            )
+        if self.struct_name in [i + "Field" for i in TYPES_ALL]:
+            raise ValidationError(
+                f"{self.struct_name} is dsdl build-in value name, please rename it. "
+                f"Build-in value names are: {', '.join(TYPES_ALL)}"
+            )
         check_name_format(self.struct_name)
 
 
-
 @dataclass()
 class StructParams:
     params_list: List[SingleStructParam] = field(default_factory=list)
@@ -38,7 +41,9 @@ class ParserParam:
     def __init__(self, sample_type, struct_defi, global_info_type=None):
         self.sample_param_map = self._parse_param_sample_type(raw=sample_type)
         if global_info_type:
-            self.global_info_param_map = self._parse_param_sample_type(raw=global_info_type)
+            self.global_info_param_map = self._parse_param_sample_type(
+                raw=global_info_type
+            )
         else:
             self.global_info_param_map = None
         self.general_param_map = self._get_params(class_defi=struct_defi)
@@ -63,7 +68,9 @@ def _parse_param_sample_type(raw: str) -> Optional[SingleStructParam]:
         else:
             return None
 
-    def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleStructParam]]:
+    def _get_params(
+        self, class_defi: Dict[str, Dict]
+    ) -> Optional[Dict[str, SingleStructParam]]:
         self.general_param_map = defaultdict(SingleStructParam)
         ###################################################################################
         # 1。对class_defi循环，先拿到每个struct的params(self.general_param_map), 需要一个单独的循环，因为定义的顺序不一定
@@ -96,34 +103,56 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
         if len(self.general_param_map) == 0:
             return None
         if len(self.general_param_map) == 1:
+            if not self.sample_param_map and not self.general_param_map:
+                raise DefineSyntaxError(
+                    f"error in definition of `sample-type` or `global-info-type`."
+                )
             temp_param_map = list(self.general_param_map.values())[0]
             temp_name = list(self.general_param_map.keys())[0]
             # 如果需要填的参数在sample-type中：
-            if temp_name == self.sample_param_map.struct_name:
+            if self.sample_param_map and temp_name == self.sample_param_map.struct_name:
                 for key in temp_param_map.params_dict.keys():
                     try:
                         self.general_param_map[temp_name].params_dict[
                             key
                         ] = self.sample_param_map.params_dict[key]
                     except KeyError as e:
-                        raise DefineSyntaxError(f"miss the params {e} in definition")
+                        raise DefineSyntaxError(
+                            f"miss the parameters {e} in {temp_name} in `sample-type`."
+                        )
             # 如果需要填的参数在global-info-type中：
-            elif self.global_info_param_map and temp_name == self.global_info_param_map.struct_name:
+            elif (
+                self.global_info_param_map
+                and temp_name == self.global_info_param_map.struct_name
+            ):
                 for key in temp_param_map.params_dict.keys():
                     try:
                         self.general_param_map[temp_name].params_dict[
                             key
                         ] = self.global_info_param_map.params_dict[key]
                     except KeyError as e:
-                        raise DefineSyntaxError(f"miss the params {e} in definition")
+                        raise DefineSyntaxError(
+                            f"miss the parameters {e} in '{temp_name}' in `global-info-type`."
+                        )
             else:
-                warnings.warn(f"parameters in struct {temp_name} is not defined", DefineSyntaxWarning)
+                raise DefineSyntaxError(
+                    f"parameter in struct `{temp_name}` is not defined correctly: "
+                    f"1. check your `sample-type` or `global-info-type`, "
+                    f"2. check the definition of struct {temp_name}."
+                )
         else:
+            if not self.sample_param_map and not self.general_param_map:
+                raise DefineSyntaxError(
+                    f"error in definition of `sample-type` or `global-info-type`."
+                )
             for define_name, define_value in class_defi.items():
                 if "$def" in define_value and define_value["$def"] == "struct":
                     struct_params = define_value.get("$params", None)
                     if struct_params:
-                        if define_name == self.sample_param_map.struct_name:
+                        if (
+                            self.sample_param_map
+                            and define_name == self.sample_param_map.struct_name
+                        ):
                             for key in self.general_param_map[define_name].params_dict:
                                 try:
                                     self.general_param_map[define_name].params_dict[
@@ -131,17 +160,22 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
                                     ] = self.sample_param_map.params_dict[key]
                                 except KeyError as e:
                                     raise DefineSyntaxError(
-                                        f"miss the params {e} in definition"
+                                        f"miss the parameters {e} in '{define_name}' in `sample-type`."
                                     )
                         # 如果需要填的参数在global-info-type中：
-                        elif self.global_info_param_map and define_name == self.global_info_param_map.struct_name:
+                        elif (
+                            self.global_info_param_map
+                            and define_name == self.global_info_param_map.struct_name
+                        ):
                             for key in self.general_param_map[define_name].params_dict:
                                 try:
                                     self.general_param_map[define_name].params_dict[
                                         key
                                     ] = self.global_info_param_map.params_dict[key]
                                 except KeyError as e:
-                                    raise DefineSyntaxError(f"miss the params {e} in definition")
+                                    raise DefineSyntaxError(
+                                        f"miss the parameters {e} in '{define_name}' in `global-info-type`."
+                                    )
 
                         for raw_type in define_value["$fields"].values():
                             field_type = raw_type.replace(" ", "")
@@ -157,15 +191,16 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
                                         temp = re.findall(
                                             r"%s\[(.*?)\]" % str(structure), field_type
                                         )[0]
-                                    except KeyError:
+                                    except (KeyError, IndexError):
                                         raise DefineSyntaxError(
-                                            f"definition error of filed {field_type}"
+                                            f"definition error of filed '{field_type}' in struct `{define_name}`, "
+                                            f"check if parameter in '{field_type}' is defined correctly."
                                         )
                                     for param in temp.split(","):
                                         param = param.split("=")
                                         if len(param) != 2:
                                             raise DefineSyntaxError(
-                                                f"error in params definition {field_type}"
+                                                f"error in parameters definition of '{field_type}' in struct `{define_name}`."
                                             )
                                         key, value = param[0], param[1]
                                         try:
@@ -174,7 +209,8 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
                                             ].params_dict[key] = value
                                         except KeyError as e:
                                             raise DefineSyntaxError(
-                                                f"miss the params {e} in definition"
+                                                f"parameter {e} of '{field_type}' in struct `{define_name}` "
+                                                f"must be defined."
                                             )
                                     break
             ###################################################################################
@@ -183,7 +219,15 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
             sort_param_dict = {}
             for key, val in self.general_param_map.items():
                 if len(val.parents_struct) > 1:  # 目前只能处理只有一个父节点的情况
-                    raise DefineSyntaxError(f"error in definition")
+                    if len(val.params_dict) <= 1 and len(set(val.parents_struct)) == 1:
+                        sort_param_dict[key] = list(set(val.parents_struct))
+                    else:
+                        raise DefineSyntaxError(
+                            f"each struct with param must have one parent struct, "
+                            f"but can have more than one child struct.\n"
+                            f"{key} have more than one parent struct"
+
+                        )
                 else:
                     sort_param_dict[key] = val.parents_struct  # {list[str]}
             ordered_keys = sort_nx(sort_param_dict)  # 用有向图排序
@@ -197,6 +241,7 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
             #     parents_struct: [SceneAndObjectSample]},}
             for struct in ordered_keys:
                 parent_struct = self.general_param_map[struct].parents_struct
+                param_dict = self.general_param_map[struct].params_dict
                 if not parent_struct:
                     # 因为加了global-info-type以后有两条线了，所以对于没有父类的情况先不处理，最后第4步再统一检查
                     continue
@@ -204,14 +249,20 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
                     #     f"each struct must have one parent struct, but {struct} have no"
                     # )
                 if len(parent_struct) > 1:
-                    raise DefineSyntaxError(
-                        "each struct must have one parent struct, but can have more than one child struct.\n"
-                        f"{struct} have more than one parent struct"
-                    )
+                    if len(param_dict) <= 1 and len(set(parent_struct)) == 1:
+                        pass
+                    else:
+                        raise DefineSyntaxError(
+                            "each struct with param must have one parent struct, "
+                            "but can have more than one child struct.\n"
+                            f"`{struct}` have more than one parent struct."
+                        )
                 parent_struct = parent_struct[0]
                 for key, val in self.general_param_map[struct].params_dict.items():
                     if val is None:
-                        raise DefineSyntaxError(f"parameter {key} of {struct} must be defined.")
+                        raise DefineSyntaxError(
+                            f"parameter {key} of {struct} must be defined."
+                        )
                     if val.startswith("$"):
                         parent_key = val.replace("$", "").strip()
                         self.general_param_map[struct].params_dict[
@@ -223,14 +274,24 @@ def _get_params(self, class_defi: Dict[str, Dict]) -> Optional[Dict[str, SingleS
             # 4。最后对self.general_param_map中所有的struct中的参数进行检验，如果有参数没有实例化，就报错
             for struct, single_struct_param in self.general_param_map.items():
                 for key, val in single_struct_param.params_dict.items():
+                    if not val:
+                        raise DefineSyntaxError(
+                            f"parameter {val} of {key} in {struct} must be defined."
+                            f"1. check your `sample-type` or `global-info-type`, "
+                            f"2. check the definition of struct {struct}."
+                        )
                     if val.startswith("$"):
-                        raise DefineSyntaxError(f"parameter {val} of {key} in {struct} must be defined.")
+                        raise DefineSyntaxError(
+                            f"parameter {val} of {key} in {struct} must be defined."
+                            f"1. check your `sample-type` or `global-info-type`, "
+                            f"2. check the definition of struct {struct}."
+                        )
         return self.general_param_map
 
     def validate_params(self, struct_params_field: Set, struct_name: str) -> Set:
         input_params = set(self.general_param_map[struct_name].params_dict.keys())
         if input_params != struct_params_field:
             raise DefineSyntaxError(
-                f"error of definition of params {struct_params_field}"
+                f"error of definition of parameters {struct_params_field} in {struct_name}."
             )
         return struct_params_field
diff --git a/dsdl/parser/parser.py b/dsdl/parser/parser.py
index 9749194..fd3d7c1 100644
--- a/dsdl/parser/parser.py
+++ b/dsdl/parser/parser.py
@@ -75,12 +75,12 @@ def __post_init__(self):
         if self.name in TYPES_ALL:
             raise ValidationError(
                 f"{self.name} is dsdl build-in value name, please rename it."
-                f"Build-in value names are: {','.join(TYPES_ALL)}"
+                f"Build-in value names are: {','.join(TYPES_ALL)}."
             )
         if self.name in [i + "Field" for i in TYPES_ALL]:
             raise ValidationError(
                 f"{self.name} is dsdl build-in value name, please rename it."
-                f"Build-in value names are: {','.join(TYPES_ALL)}"
+                f"Build-in value names are: {','.join(TYPES_ALL)}."
             )
         check_name_format(self.name)
 
@@ -116,7 +116,7 @@ def _parse(self, data_file: str, library_path: str):
         try:
             self.meta = desc["meta"]  # 存版meta信息，后续应该会使用（目前木有用）
         except KeyError as e:
-            err_msg = f"data yaml must contains {e} section"
+            err_msg = f"data yaml must contains {e} section."
             if self.report_flag:
                 temp_check_item = CheckLogItem(
                     def_name="all", msg=f"DefineSyntaxError: {err_msg}"
@@ -130,7 +130,7 @@ def _parse(self, data_file: str, library_path: str):
             data_sample_type = desc["data"]["sample-type"]
         except KeyError as e:
             err_msg = (
-                f"data yaml must contains {e} in `data` section with `sample-type`"
+                f"data yaml must contain `data` section and `data` section must have `sample-type`."
             )
             if self.report_flag:
                 temp_check_item = CheckLogItem(
@@ -145,7 +145,7 @@ def _parse(self, data_file: str, library_path: str):
             global_info_type = desc["data"]["global-info-type"]
         except KeyError as e:
             global_info_type = None
-            warning_msg = f"{e}, `global-info-type` is not defined"
+            warning_msg = f"{e}, `global-info-type` is not defined."
             if self.report_flag:
                 temp_check_item = CheckLogItem(
                     def_name="all", msg=f"DefineSyntaxWarning: {warning_msg}"
@@ -164,8 +164,8 @@ def _parse(self, data_file: str, library_path: str):
                         import_list.append(temp_p)
                     else:
                         raise DSDLImportError(
-                            f"{p} is not exists in `{library_path}`,"
-                            f"please given right path using `-p`."
+                            f"{p} does not exist in `{library_path}`, "
+                            f"please check the path or give the right path using `-p`."
                         )
             else:
                 library_path = os.path.dirname(data_file)
@@ -181,8 +181,8 @@ def _parse(self, data_file: str, library_path: str):
                             import_list.append(temp_p)
                         else:
                             raise DSDLImportError(
-                                f"{p} is not exists in neither `{library_path}` nor `dsdl/dsdl_library`,"
-                                f"please given right path using `-p`."
+                                f"{p} does not exist in neither `{library_path}` nor `dsdl/dsdl_library`,"
+                                f"please check the path or give the right path using `-p`."
                             )
 
         if "defs" in desc:
@@ -229,7 +229,7 @@ def _parse(self, data_file: str, library_path: str):
             try:
                 define_type = define_value["$def"]
             except KeyError as e:
-                err_msg = f"{define_name} section must contains {e} sub-section"
+                err_msg = f"{define_name} section must contain {e} sub-section."
                 if self.report_flag:
                     temp_check_item = CheckLogItem(
                         def_name="all", msg=f"DefineSyntaxError: {err_msg}"
@@ -304,8 +304,9 @@ def _parse(self, data_file: str, library_path: str):
                 # deal with `$optional` section after `$fields` section，
                 # because we must ensure filed in `$optional` is the `filed_name` in `$fields` section.
                 if "$optional" in define_value or FIELD_PARSER.optional:
+                    temp = define_value.get("$optional", set())
                     optional_set = (
-                        set(define_value["$optional"]) | FIELD_PARSER.optional
+                        set(temp) | FIELD_PARSER.optional
                     )
                     for optional_name in optional_set:
                         optional_name = optional_name.strip()
diff --git a/dsdl/parser/utils.py b/dsdl/parser/utils.py
index 3a0e7e6..6e48488 100644
--- a/dsdl/parser/utils.py
+++ b/dsdl/parser/utils.py
@@ -28,19 +28,36 @@
 TYPES_LIST = ["List"]
 TYPES_IMAGE_SHAPE = ["ImageShape"]
 TYPES_ROTATED_BBOX = ["RotatedBBox"]
-TYPES_ALL = TYPES_WITHOUT_PARS + TYPES_TIME + TYPES_LABEL + TYPES_LIST + TYPES_IMAGE_SHAPE + TYPES_ROTATED_BBOX
+TYPES_UNIQUE_ID = ["UniqueID"]
+TYPES_ALL = (
+    TYPES_WITHOUT_PARS
+    + TYPES_TIME
+    + TYPES_LABEL
+    + TYPES_LIST
+    + TYPES_IMAGE_SHAPE
+    + TYPES_ROTATED_BBOX
+    + TYPES_UNIQUE_ID
+)
 
 
 class CheckLogItem(models.Base):
-    def_name = fields.StringField(required=True, validators=validators.Enum("class_domain", "struct", "all"))
+    def_name = fields.StringField(
+        required=True, validators=validators.Enum("class_domain", "struct", "all")
+    )
     yaml = fields.StringField(nullable=True)
-    flag = fields.IntField(required=True, validators=validators.Enum(0, 1), default=0)  # 0:error, 1: right
+    flag = fields.IntField(
+        required=True, validators=validators.Enum(0, 1), default=0
+    )  # 0:error, 1: right
     msg = fields.StringField(nullable=True)
 
 
 class CheckLog(models.Base):
-    def_name = fields.StringField(required=True, validators=validators.Enum("class_domain", "struct", "all"))
-    flag = fields.IntField(required=True, validators=validators.Enum(0, 1), default=0)  # 0:error, 1: right
+    def_name = fields.StringField(
+        required=True, validators=validators.Enum("class_domain", "struct", "all")
+    )
+    flag = fields.IntField(
+        required=True, validators=validators.Enum(0, 1), default=0
+    )  # 0:error, 1: right
     sub_struct = fields.ListField([CheckLogItem], nullable=True)
 
 
@@ -58,7 +75,7 @@ def sanitize_variable_name(varstr: str) -> str:
 def check_name_format(varstr: str):
     if not varstr.isidentifier():
         err_msg = (
-            f"`{varstr}` must be a a valid identifier. "
+            f"`{varstr}` must be a valid identifier. "
             f"[1. `Struct` name 2. `Class domain` name 3.name of `$field` in `Struct`] "
             f"is considered a valid identifier if "
             f"it only contains alphanumeric letters (a-z) and (0-9), or underscores (_). "
@@ -75,6 +92,28 @@ def check_name_format(varstr: str):
         raise ValidationError(err_msg)
 
 
+def check_is_bracket_pair(var_str: str) -> bool:
+    """
+    check if var_str has bracket in pairs and in order, return True: yes, False: not in pairs or in order
+    """
+    bracket = {")": "(", "]": "[", "}": "{"}
+    b = []
+    for i in var_str:
+        if i in bracket.values():
+            b.append(i)
+        elif len(b) > 0 and b[-1] == bracket.get(i):
+            b.pop()
+        elif i in bracket.keys():
+            b.append(i)
+        else:
+            pass
+    if len(b) == 0:
+        return True
+    else:
+        return False
+
+
+
 def rreplace(s, old, new, occurrence):
     """
     从右向左的替换函数，类似replace,不过是反着的
@@ -120,3 +159,5 @@ def sort_nx(
         raise "define cycle found."
     ordered_keys = list(nx.topological_sort(define_graph))
     return ordered_keys
+
+
diff --git a/dsdl/tools/check.py b/dsdl/tools/check.py
index c35fdc1..ab40889 100644
--- a/dsdl/tools/check.py
+++ b/dsdl/tools/check.py
@@ -16,7 +16,7 @@
 @click.command(name="check")
 @click.option("-y", "--yaml", "dsdl_yaml", type=str, required=True, help="the path of dsdl yaml file")
 @click.option("-c", "--config", "config", type=str, required=True, help="the path of the config file")
-@click.option("-l", "--location", "location", type=click.Choice(["local", "ali-oss"]), required=True,
+@click.option("-l", "--location", "location", type=click.Choice(["local", "ali-oss", "ceph"]), required=True,
               help="the path of the config file")
 @click.option("-n", "--num", "num", type=int, default=5, help="how many samples sampled from the dataset")
 @click.option("-r", "--random", is_flag=True, help="whether to sample randomly")
@@ -38,10 +38,21 @@ def check(dsdl_yaml, num, random, fields, config, position, output, **kwargs):
         parse_report = check_dsdl_parser(dsdl_yaml["yaml_file"], dsdl_library_path="", report_flag=True)
     dsdl_py = parse_report["dsdl_py"]
     parse_report = json.loads(parse_report["check_log"])
+    parse_report["samples"] = {
+        "flag": 1,
+        "msg": f"Totally {len(dsdl_yaml['samples'])} samples found."
+    }
+    if len(dsdl_yaml["samples"]) == 0:
+        parse_report['flag'] = 0
+        parse_report["samples"] = {
+            "flag": 0,
+            "msg": "No samples found, please check the path of json file."
+        }
     report_obj.set_parser_info(parse_report)
     if parse_report['flag'] == 0:
         report_obj.generate()
         return
+
     exec(dsdl_py, {})
 
     dataset = CheckDataset(report_obj, dsdl_yaml["samples"], dsdl_yaml["sample_type"], config,
diff --git a/dsdl/tools/commons.py b/dsdl/tools/commons.py
index bccedb4..5e9db1f 100644
--- a/dsdl/tools/commons.py
+++ b/dsdl/tools/commons.py
@@ -142,7 +142,7 @@ def process(dsdl_yaml, config, location, num, random, fields, task, position, **
             config_dic = {}
             with open(config, encoding='utf-8') as config_file:
                 exec(config_file.read(), config_dic)
-            location_config = config_dic["local" if location == "local" else "ali_oss"]
+            location_config = config_dic[location.replace("-", "_")]
             result["config"] = location_config
 
             # parse field list
diff --git a/dsdl/types/__init__.py b/dsdl/types/__init__.py
index 11047c8..e4602cd 100644
--- a/dsdl/types/__init__.py
+++ b/dsdl/types/__init__.py
@@ -14,6 +14,7 @@
     TextField,
     ImageShapeField,
     InstanceIDField,
+    UniqueIDField,
 )
 from .unstructure import ImageField, LabelMapField, InstanceMapField
 
@@ -41,5 +42,6 @@
     "TextField",
     "ImageShapeField",
     "InstanceIDField",
+    "UniqueIDField",
     "StructMetaclass",
 ]
diff --git a/dsdl/types/field.py b/dsdl/types/field.py
index 99e6669..1612118 100644
--- a/dsdl/types/field.py
+++ b/dsdl/types/field.py
@@ -1,12 +1,7 @@
 class Field:
-    def __init__(self, is_attr=False, optional=False):
-        self._is_attr = is_attr
+    def __init__(self, optional=False):
         self._optional = optional
 
-    @property
-    def is_attr(self):
-        return self._is_attr
-
     @property
     def is_optional(self):
         return self._optional
diff --git a/dsdl/types/generic.py b/dsdl/types/generic.py
index 2db503b..1189708 100644
--- a/dsdl/types/generic.py
+++ b/dsdl/types/generic.py
@@ -39,20 +39,50 @@ def __init__(self, ele_type, ordered=False, *args, **kwargs):
         self.ordered = ordered
         self.ele_type = ele_type
         self.file_reader = None
+        self.prefix = None
+        self.flatten_dic = None
+        self.lazy_init = False
         super().__init__(*args, **kwargs)
 
     def validate(self, value):
         if hasattr(self.ele_type, "set_file_reader"):
             self.ele_type.set_file_reader(self.file_reader)
         if isinstance(self.ele_type, Field):
-            value = [self.ele_type.validate(item) for item in value]
+            if self.prefix is not None and self.flatten_dic is not None:
+                field_key = self.ele_type.extract_key()
+                field_dic = self.flatten_dic.setdefault(field_key, {})
+                res = [self.ele_type.validate(_) for _ in value]
+                paths = [f"{self.prefix}/{ind}" for ind in range(len(res))]
+                _ = [field_dic.update({k: v}) for k, v in zip(paths, res)]
+            else:
+                res = [self.ele_type.validate(_) for _ in value]
+
         elif isinstance(self.ele_type, Struct):
-            value = [self.ele_type.__class__(file_reader=self.file_reader, **item) for item in value]
-        return value
+            res = [
+                self.ele_type.__class__(
+                    file_reader=self.file_reader,
+                    prefix=f"{self.prefix}/{i}",
+                    flatten_dic=self.flatten_dic,
+                    lazy_init=self.lazy_init, **item)
+                for i, item in enumerate(value)
+            ]
+        else:
+            raise ValidationError(
+                f"Wrong ele_type in ListField, only 'Struct' or 'Field' are permitted, but got '{type(self.ele_type)}'")
+        return res
 
     def set_file_reader(self, file_reader):
         self.file_reader = file_reader
 
+    def set_prefix(self, prefix):
+        self.prefix = prefix
+
+    def set_flatten_dic(self, dic):
+        self.flatten_dic = dic
+
+    def set_lazy_init(self, value):
+        self.lazy_init = value
+
 
 class DictField(Field):
     def validate(self, value):
diff --git a/dsdl/types/special.py b/dsdl/types/special.py
index 7deb044..20b346e 100644
--- a/dsdl/types/special.py
+++ b/dsdl/types/special.py
@@ -1,5 +1,5 @@
 from .field import Field
-from ..geometry import BBox, Polygon, PolygonItem, Coord2D, KeyPoints, Text, RBBox, ImageShape, InstanceID
+from ..geometry import BBox, Polygon, PolygonItem, Coord2D, KeyPoints, Text, RBBox, ImageShape, UniqueID
 from ..exception import ValidationError
 from datetime import date, time, datetime
 import math
@@ -63,7 +63,7 @@ def validate(self, value):
                 value[-1] = value[-1] / 180 * math.pi  # convert to radian
         else:
             value = validate_list_of_number(value, 8, float, "RotateBBoxField")
-            value = [value[i:i+2] for i in (0, 2, 4, 6)]
+            value = [value[i:i + 2] for i in (0, 2, 4, 6)]
         return RBBox(value, self.mode)
 
 
@@ -184,6 +184,18 @@ def validate(self, value):
 class InstanceIDField(Field):
     def validate(self, value):
         try:
-            return InstanceID("" + value)
+            return UniqueID("" + value, "InstanceID")
         except TypeError as _:
             raise ValidationError(f"InstanceIDField Error: expect str value, got {value}")
+
+
+class UniqueIDField(Field):
+    def __init__(self, id_type=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.id_type = id_type
+
+    def validate(self, value):
+        try:
+            return UniqueID("" + value, self.id_type)
+        except TypeError as _:
+            raise ValidationError(f"UniqueIDField Error: expect str value, got {value}")
diff --git a/dsdl/types/struct.py b/dsdl/types/struct.py
index 278c023..45ec606 100644
--- a/dsdl/types/struct.py
+++ b/dsdl/types/struct.py
@@ -1,13 +1,118 @@
+import re
+import os
+from fnmatch import translate
 from .field import Field
-from ..geometry import Attributes, STRUCT
+from ..geometry import STRUCT
 from ..exception import ValidationError
 from ..warning import FieldNotFoundWarning
+import functools
+
+
+def _is_magic(p):
+    return "[" in p or "]" in p or "*" in p or "?" in p
+
+
+class RegisterPattern:
+    """
+    {
+        './a/b/*/[1-9]/*': {
+            "$bbox": [('./a/b/det/%d/box', 1), ('./a/b/ann/%d/box', 1)]
+            "$label": [('./a/b/det/[1-9]/cate', 1), ('./a/b/ann/[1-9]/cate', 1)]
+        }
+    }
+    """
+
+    def __init__(self):
+        self.flatten_struct = None
+        self.registered_patterns = dict()
+
+    def set_flatten_struct(self, flatten_struct):
+        self.flatten_struct = flatten_struct
+
+    def register_pattern(self, pattern):
+        if not _is_magic(pattern):
+            return
+        assert self.flatten_struct is not None
+        if pattern in self.registered_patterns:
+            return
+        res_dic = self.registered_patterns.setdefault(pattern, {})
+        pattern = os.path.normcase(pattern)
+        pattern_seg = pattern.split(os.sep)
+        pattern_seg = [(re.compile(translate(_)), _) if _is_magic(_) else _ for _ in pattern_seg]
+        for field_key, field_info in self.flatten_struct.items():
+            if field_key == "$field_mapping":
+                continue
+            field_lst = res_dic.setdefault(field_key, [])
+            for field_path in field_info:
+                match_res = self._match(field_path, pattern_seg)
+                field_lst.append(match_res) if match_res else None
+
+    def get_parsed_pattern(self, pattern, field_keys=None):
+        patterns_res = self.registered_patterns[pattern]
+        if field_keys is None:
+            return patterns_res
+
+        res = dict()
+        for field_key in field_keys:
+            field_info = patterns_res.get(field_key, None)
+            if field_info is not None:
+                res[field_key] = field_info
+        return res
+
+    def has_registered(self, pattern):
+        return pattern in self.registered_patterns
+
+    @staticmethod
+    def _match(path, pattern_seg):
+        path = os.path.normcase(path)
+        path_seg = path.split(os.sep)
+        magic_num = 0
+        if len(path_seg) != len(pattern_seg):
+            return False
+        for i, (pattern_, path_) in enumerate(zip(pattern_seg, path_seg)):
+            if path_ != "*":  # not list
+                if isinstance(pattern_, tuple):
+                    p_compile, p_str = pattern_
+                    if p_compile.match(path_) is None:
+                        return False
+                else:
+                    if pattern_ != path_:
+                        return False
+            else:  # list
+                if isinstance(pattern_, tuple):
+                    path_seg[i] = "%d"
+                    magic_num += 1
+                else:
+                    if pattern_.isdigit():
+                        path_seg[i] = pattern_
+                    else:
+                        return False
+
+        return "/".join(path_seg), magic_num
+
+
+def lazy_init_wrapper(f):
+    @functools.wraps(f)
+    def func(self, *args, **kwargs):
+        assert self.lazy_init, "This method can used only when 'lazy_init' mode is turned on."
+        return f(self, *args, **kwargs)
+
+    return func
+
+
+def no_lazy_init_wrapper(f):
+    @functools.wraps(f)
+    def func(self, *args, **kwargs):
+        assert not self.lazy_init, "This method can used only when 'lazy_init' mode is turned off."
+        return f(self, *args, **kwargs)
+
+    return func
 
 
 class StructMetaclass(type):
     def __new__(mcs, name, bases, attributes):
-        super_new = super().__new__
 
+        super_new = super().__new__
         # Also ensure initialization is only performed for subclasses of Model
         # (excluding Model class itself).
         parents = [b for b in bases if isinstance(b, StructMetaclass)]
@@ -16,15 +121,12 @@ def __new__(mcs, name, bases, attributes):
 
         required = dict()  # 所有必须填写的field对象
         optional = dict()  # 所有可不填写的field对象
-        attr = dict()  # 所有是attribute的field对象
         mappings = dict()  # 所有的field对象
         struct_mappings = dict()  # 所有的Struct对象
 
         for k, v in attributes.items():
             if isinstance(v, Field):
                 mappings[k] = v
-                if v.is_attr:
-                    attr[k] = v
                 if v.is_optional:
                     optional[k] = v
                 else:
@@ -40,7 +142,6 @@ def __new__(mcs, name, bases, attributes):
             attributes.pop(k)
 
         attributes["__required__"] = required
-        attributes["__attr__"] = attr
         attributes["__optional__"] = optional
         attributes["__mappings__"] = mappings
         attributes["__struct_mappings__"] = struct_mappings
@@ -51,94 +152,280 @@ def __new__(mcs, name, bases, attributes):
 
 
 class Struct(dict, metaclass=StructMetaclass):
-    def __init__(self, file_reader=None, **kwargs):
+    _FLATTEN_STRUCT = None
+    _REGISTER_PATTERN = RegisterPattern()
+    _FILE_READER = None
 
+    def __init__(self, file_reader=None, prefix=None, flatten_dic=None, lazy_init=False, **kwargs):
         super().__init__()
-        self.attributes = Attributes()
         self.file_reader = file_reader
-        self._keys = []
-        self._dict_format = None
-        if file_reader is not None:  # 说明是在赋值
+        self._raw_dict = kwargs
+        self.lazy_init = lazy_init
+        if lazy_init:
+            self._set_file_reader(file_reader)
+            self.init_pattern_register()
+        else:
+            assert flatten_dic is None or isinstance(flatten_dic, dict)
+            self._keys = []
+            self._dict_format = None
+            self._flatten_format = dict() if flatten_dic is None else flatten_dic
+            self._prefix = prefix or "."
+
             for k in self.__required__:
                 if k not in kwargs:
                     FieldNotFoundWarning(f"Required field {k} is missing.")
                     continue
                 setattr(self, k, kwargs[k])
-                if k not in self.__attr__:
+                self._keys.append(k)
+            for k in self.__optional__:
+                if k in kwargs:
+                    setattr(self, k, kwargs[k])
                     self._keys.append(k)
-        for k in self.__optional__:
-            if k in kwargs:
+            for k in self.__struct_mappings__:
+                if k not in kwargs:
+                    FieldNotFoundWarning(f"Required struct instance {k} is missing.")
+                    continue
                 setattr(self, k, kwargs[k])
-                if k not in self.__attr__:
-                    self._keys.append(k)
-        for k in self.__struct_mappings__:
-            if k not in kwargs:
-                FieldNotFoundWarning(f"Required struct instance {k} is missing.")
-                continue
-            setattr(self, k, kwargs[k])
-            self._keys.append(k)
+                self._keys.append(k)
 
-        self["$attributes"] = self.attributes
-        self._keys.append("$attributes")
+    @classmethod
+    def _set_file_reader(cls, file_reader):
+        cls._FILE_READER = file_reader
 
     def __getattr__(self, key):
         try:
             return self[key]
         except KeyError:
-            raise AttributeError(r"'Model' object has no attribute '%s'" % key)
+            if not self["lazy_init"]:
+                raise AttributeError(r"'Model' object has no attribute '%s'" % key)
+            else:
+                try:
+                    value = self["_raw_dict"][key]
+                except KeyError:
+                    raise AttributeError(f"Key '{key}' doesn't exist in the current sample.")
+                field = self.get_mapping().get(key, None)
+                if field is not None:
+                    if hasattr(field, "set_file_reader"):
+                        field.set_file_reader(self["file_reader"])
+                    if hasattr(field, "set_lazy_init"):
+                        field.set_lazy_init(self["lazy_init"])
+                    return field.validate(value)
+                struct = self.get_struct_mapping().get(key, None)
+                if struct is not None:
+                    return struct.__class__(file_reader=self["file_reader"], lazy_init=self["lazy_init"], **value)
+                return value
 
     def __setattr__(self, key, value):
 
-        if key in self.__mappings__:
-            if hasattr(self.__mappings__[key], "set_file_reader"):
-                self.__mappings__[key].set_file_reader(self.file_reader)
+        if key in self.__mappings__:  # field
+            field_obj = self.__mappings__[key]
+            if hasattr(field_obj, "set_file_reader"):
+                field_obj.set_file_reader(self.file_reader)
+            if hasattr(field_obj, "set_prefix"):
+                field_obj.set_prefix(f"{self._prefix}/{key}")
+            if hasattr(field_obj, "set_flatten_dic"):
+                field_obj.set_flatten_dic(self._flatten_format)
             try:
-                if key in self.__attr__:
-                    self.attributes[key] = self.__mappings__[key].validate(value)
-                    return
-                self[key] = self.__mappings__[key].validate(value)
+                geometry_obj = field_obj.validate(value)
             except ValidationError as error:
                 raise ValidationError(f"Field '{key}' validation error: {error}.")
-        elif key in self.__struct_mappings__:
+            self[key] = geometry_obj
+            field_key = field_obj.extract_key()
+            if field_key != "$list":
+                path = f"{self._prefix}/{key}"
+                tmp_field_dic = self._flatten_format.setdefault(field_key, {})
+                tmp_field_dic[path] = geometry_obj
+
+        elif key in self.__struct_mappings__:  # struct
             cls = self.__struct_mappings__[key].__class__
             if not isinstance(value, dict):
                 raise ValidationError(
                     f"Struct validation error: {cls.__name__} requires a dict to initiate, but got '{value}'.")
-            self[key] = cls(file_reader=self.file_reader, **value)
+            struct_obj = cls(
+                file_reader=self.file_reader,
+                prefix=f"{self._prefix}/{key}",
+                flatten_dic=self._flatten_format, **value)
+            self[key] = struct_obj
         else:
             self[key] = value
-            return
 
-    def get_mapping(self):
-        return self.__mappings__
+    @classmethod
+    def get_mapping(cls):
+        return cls.__mappings__
 
-    def get_struct_mapping(self):
-        return self.__struct_mappings__
+    @classmethod
+    def get_struct_mapping(cls):
+        return cls.__struct_mappings__
 
     def keys(self):
-        return tuple(self._keys)
+        if not self.lazy_init:
+            return tuple(self._keys)
+        else:
+            return ("lazy_init", "_raw_dict", "file_reader",)
 
-    def convert2dict(self):
-        """
-        Convert struct instance to a dict like:
-             sample = {
-                "$image": {"img2": img_obj, "img1":img_obj},
-                "$list": {"objects": [{"$image": {"img1": img_obj}, "$bbox": {"box": box_obj}}],
-                            "object2": [{"$bbox":{"box": box_obj}}, {"$bbox":{"box": box_obj}}], },
-                "$struct": {
-                    "struct1": {
-                        "$image": {"img3": img_obj, "img4": img_obj}
-                    "struct2": {
-                        "$image": {"img4": img_obj}}
-                    }
-                }
-            }
-        """
-        if self._dict_format is None:
-            self._dict_format = self._parse_struct(self)
-        return self._dict_format
+    def extract_path_info(self, pattern, field_keys=None, verbose=False):
+        if self.lazy_init:
+            return self.lazy_extract_path_info(pattern, field_keys, verbose)
+        else:
+            return self.no_lazy_extract_path_info(pattern, field_keys, verbose)
 
-    def extract_field_info(self, field_lst):
+    def extract_field_info(self, field_lst, nest_flag=True, verbose=False):
+        if self.lazy_init:
+            return self.lazy_extract_field_info(field_lst, nest_flag, verbose)
+        else:
+            return self.no_lazy_extract_field_info(field_lst, nest_flag, verbose)
+
+    @lazy_init_wrapper
+    def lazy_extract_path_info(self, pattern, field_keys=None, verbose=False):
+        if field_keys is not None:
+            if not isinstance(field_keys, list):
+                field_keys = [field_keys]
+            field_keys = [f"${_.lower()}" for _ in field_keys if isinstance(_, str)]
+
+        if not _is_magic(pattern):  # 无通配
+            res = self.lazy_extract_path_value(pattern, field_keys)
+            if res is not None:
+                return res if verbose else list(res.values())
+            return dict() if verbose else []
+
+        self.register_path_for_extract(pattern)
+        all_parsed_pattern = self._REGISTER_PATTERN.get_parsed_pattern(pattern, field_keys)
+        res = dict()
+        for field_key in field_keys or all_parsed_pattern.keys():
+            pattern_field_info = all_parsed_pattern.get(field_key, None)
+            if pattern_field_info is None:
+                continue
+            for this_pattern in pattern_field_info:
+                this_res = self.lazy_extract_path_value(this_pattern[0].replace("%d", "*"), [field_key])
+                if this_res is not None:
+                    res.update(this_res)
+        if not verbose:
+            res = list(res.values())
+        return res
+
+    @lazy_init_wrapper
+    def lazy_extract_path_value(self, path, field_keys=None):
+
+        def _extract_value_from_pattern(p_segs, value_dic, ret_dic, prefix="."):
+            if len(p_segs) == 0:
+                ret_dic[prefix] = field_obj.validate(value_dic)
+                return
+            p = p_segs[0]
+            if p.isdigit():
+                p = int(p)
+            if p != "*":
+                _extract_value_from_pattern(p_segs[1:], value_dic[p], ret_dic, f"{prefix}/{p}")
+            else:
+                for i, v in enumerate(value_dic):  # list
+                    _extract_value_from_pattern(p_segs[1:], v, ret_dic, f"{prefix}/{i}")
+
+        path_segs = path.split("/")
+        field_path = "/".join([_ if not _.isdigit() else "*" for _ in path_segs])
+        try:
+            field_obj = self._FLATTEN_STRUCT["$field_mapping"][field_path]
+            if field_keys is not None and field_obj.extract_key() not in field_keys:
+                return None
+        except KeyError as e:
+            FieldNotFoundWarning(f"No field of path '{field_path}' exists in this struct.")
+            return None
+        value = self._raw_dict
+        res = dict()
+        try:
+            _extract_value_from_pattern(path_segs[1:], value, res)
+        except KeyError as e:
+            return None
+        except IndexError as e:
+            return None
+        except ValidationError as e:
+            raise e  # not valid
+        return res
+
+    @no_lazy_init_wrapper
+    def no_lazy_extract_path_info(self, pattern, field_keys=None, verbose=False):
+
+        if field_keys is None:
+            flatten_sample = self.flatten_sample()
+        else:
+            if not isinstance(field_keys, list):
+                field_keys = [field_keys]
+            field_keys = [f"${_.lower()}" for _ in field_keys if isinstance(_, str)]
+            all_flatten_sample = self.flatten_sample()
+            flatten_sample = dict()
+            for field_key in field_keys:
+                field_info = all_flatten_sample.get(field_key, None)
+                if field_info is not None:
+                    flatten_sample[field_key] = field_info
+        if not _is_magic(pattern):  # 无通配
+            for field_info in flatten_sample.values():
+                res = field_info.get(pattern, None)
+                if res is not None:
+                    return {pattern: res} if verbose else [res]
+            return dict() if verbose else []
+
+        self.register_path_for_extract(pattern)
+
+        all_parsed_pattern = self._REGISTER_PATTERN.get_parsed_pattern(pattern, field_keys)
+        res = dict()
+        for field_key in field_keys or all_parsed_pattern.keys():
+            pattern_field_info = all_parsed_pattern.get(field_key, None)
+            field_info = flatten_sample.get(field_key, None)
+            if pattern_field_info is None or field_info is None:
+                continue
+            for this_pattern in pattern_field_info:
+                self._match(this_pattern, field_info, res)
+        if not verbose:
+            res = list(res.values())
+        return res
+
+    @staticmethod
+    def _match(pattern, all_sample, result_dic):
+        pattern, digit_num = pattern
+        if digit_num == 0:
+            result_dic[pattern] = all_sample[pattern]
+            return
+
+        def _helper(prefix, idx_num):
+            if idx_num == 0:
+                this_pattern = pattern % tuple(prefix)
+                item = all_sample.get(this_pattern, None)
+                if item is not None:
+                    result_dic[this_pattern] = item
+                    return True
+                else:
+                    return False
+            start_idx = 0
+            while 1:
+                res = _helper(prefix + [start_idx], idx_num - 1)
+                if res:
+                    start_idx += 1
+                else:
+                    break
+            return start_idx > 0
+
+        _helper([], digit_num)
+
+    @lazy_init_wrapper
+    def lazy_extract_field_info(self, field_lst, nest_flag=True, verbose=False):
+        ori_field_lst = field_lst
+        # {field_type: [pattern1, pattern2, ...]}
+        field_paths = {_: self._FLATTEN_STRUCT.get(f"${_.lower()}", []) for i, _ in enumerate(ori_field_lst)}
+        res = dict()
+        for field, pattern_lst in field_paths.items():
+            if verbose:
+                this_res = res.setdefault(field, dict())
+            else:
+                this_res = res.setdefault(field, list())
+            for pattern in pattern_lst:
+                extract_res = self.lazy_extract_path_value(pattern, field_keys=[f"${field.lower()}"])
+                if extract_res is not None:
+                    if verbose:
+                        this_res.update(extract_res)
+                    else:
+                        this_res.extend(extract_res.values())
+        return res
+
+    @no_lazy_init_wrapper
+    def no_lazy_extract_field_info(self, field_lst, nest_flag=True, verbose=False):
         """
         Extract the field info given field list, for example, if field_lst is [bbox, image], the result will be:
 
@@ -155,17 +442,102 @@ def extract_field_info(self, field_lst):
                 }
             }
         """
+        flatten_sample = self.flatten_sample()
         result_dic = {}
-        for field_name in field_lst:
-            result_dic[field_name] = self._flatten_sample(f"${field_name}")
+        ori_field_lst = field_lst
+        field_lst = [_.lower() for _ in ori_field_lst]
+        for ori_field, field in zip(ori_field_lst, field_lst):
+            if nest_flag:
+                if verbose:
+                    result_dic[ori_field] = flatten_sample.get(f"${field}", {})
+                else:
+                    field_info = list(flatten_sample.get(f"${field}", {}).values())
+                    result_dic[ori_field] = field_info
+            else:
+                result_dic.update(flatten_sample.get(f"${field}", {}))
+        if not nest_flag and not verbose:
+            result_dic = list(result_dic.values())
         return result_dic
 
-    def _flatten_sample(self, field_name, parse_method=lambda _: _):
+    @no_lazy_init_wrapper
+    def flatten_sample(self):
+        if self._flatten_format:
+            return self._flatten_format
         result_dic = {}
-        prefix = "."
-        self._parse_helper(self.convert2dict(), field_name, result_dic, prefix, parse_method)
+        if self._dict_format is None:
+            """
+            self._dict_format: 
+                Convert struct instance to a dict like:
+                     sample = {
+                        "$image": {"img2": img_obj, "img1":img_obj},
+                        "$list": {"objects": [{"$image": {"img1": img_obj}, "$bbox": {"box": box_obj}}],
+                                    "object2": [{"$bbox":{"box": box_obj}}, {"$bbox":{"box": box_obj}}], },
+                        "$struct": {
+                            "struct1": {
+                                "$image": {"img3": img_obj, "img4": img_obj}
+                            "struct2": {
+                                "$image": {"img4": img_obj}}
+                            }
+                        }
+                    }
+            """
+            self._dict_format = self._parse_struct(self)
+        self._parse_helper(self._dict_format, result_dic)
+        self._flatten_format = result_dic
         return result_dic
 
+    def convert2json(self):
+        return self._raw_dict
+
+    @classmethod
+    def _flatten_struct(cls):
+        prefix = "."
+        res_dic = dict()
+        res_dic["$field_mapping"] = dict()
+        field_mappings = cls.get_mapping()
+        struct_mappings = cls.get_struct_mapping()
+
+        def _helper(item, pre_path, key_name, flatten_dic):
+            if isinstance(item, Field):
+                field_key = item.extract_key()
+                if field_key != "$list":
+                    path = f"{pre_path}/{key_name}"
+                    field_dic = flatten_dic.setdefault(field_key, [])
+                    field_dic.append(path)
+                    if hasattr(item, "set_file_reader"):
+                        item.set_file_reader(cls._FILE_READER)
+                    flatten_dic["$field_mapping"][path] = item
+                else:
+                    path = f"{pre_path}/{key_name}"
+                    _helper(item.ele_type, path, "*", flatten_dic)
+            elif isinstance(item, Struct):
+                path = f"{pre_path}/{key_name}"
+                this_mapping = item.get_mapping()
+                this_struct_mapping = item.get_struct_mapping()
+                for this_key, this_item in this_mapping.items():
+                    _helper(this_item, path, this_key, flatten_dic)
+                for this_key, this_item in this_struct_mapping.items():
+                    _helper(this_item, path, this_key, flatten_dic)
+
+        for field_name, field_obj in field_mappings.items():
+            _helper(field_obj, prefix, field_name, res_dic)
+        for struct_name, struct_obj in struct_mappings.items():
+            _helper(struct_obj, prefix, struct_name, res_dic)
+
+        return res_dic
+
+    @classmethod
+    def register_path_for_extract(cls, pattern):
+        cls.init_pattern_register()
+        if not cls._REGISTER_PATTERN.has_registered(pattern):
+            cls._REGISTER_PATTERN.register_pattern(pattern)
+
+    @classmethod
+    def init_pattern_register(cls):
+        if getattr(cls, "_FLATTEN_STRUCT", None) is None:
+            cls._FLATTEN_STRUCT = cls._flatten_struct()
+            cls._REGISTER_PATTERN.set_flatten_struct(cls._FLATTEN_STRUCT)
+
     @classmethod
     def _parse_struct(cls, sample):
         if isinstance(sample, Struct):
@@ -173,14 +545,8 @@ def _parse_struct(cls, sample):
             field_mapping = sample.get_mapping()  # all fields
             struct_mapping = sample.get_struct_mapping()  # all structs
             for key in sample.keys():
-                if key.startswith("$"):  # attributes
-                    field_key = key
-                    key = key.replace("$", "")
-                    if field_key in data_item:
-                        data_item[field_key][key] = getattr(sample, field_key)
-                    else:
-                        data_item[field_key] = {key: getattr(sample, field_key)}
-                elif key in field_mapping:  # fields
+
+                if key in field_mapping:  # fields
                     field_obj = getattr(sample, key)
                     field_key = field_mapping[key].extract_key()
                     if field_key in data_item:
@@ -202,25 +568,28 @@ def _parse_struct(cls, sample):
             return sample
 
     @classmethod
-    def _parse_helper(cls, sample, field_name, result_dic, prefix=".", parse_method=lambda _: _):
-        assert field_name not in ("$struct", "$list")
+    def _parse_helper(cls, sample, result_dic, field_name=None, prefix=".", parse_method=lambda _: _):
 
         if isinstance(sample, dict):
             for field_type in sample:
-                if field_type == field_name:
+                if (field_name is None and field_type not in ("$struct", "$list")) or (
+                        field_name is not None and field_type in field_name):
                     for key, value in sample[field_type].items():
                         k_ = f"{prefix}/{key}"
-                        result_dic[k_] = parse_method(value)
-                if field_type == "$list":
+                        tmp_dic = result_dic.setdefault(field_type, {})
+                        tmp_dic[k_] = parse_method(value)
+                elif field_type == "$list":
                     for key, value in sample[field_type].items():
                         k_ = f"{prefix}/{key}"
-                        cls._parse_helper(value, field_name, result_dic, prefix=k_, parse_method=parse_method)
-                if field_type == "$struct":
+                        cls._parse_helper(value, result_dic, field_name=field_name, prefix=k_,
+                                          parse_method=parse_method)
+                elif field_type == "$struct":
                     for key, value in sample[field_type].items():
                         k_ = f"{prefix}/{key}"
-                        cls._parse_helper(value, field_name, result_dic, prefix=k_, parse_method=parse_method)
+                        cls._parse_helper(value, result_dic, field_name=field_name, prefix=k_,
+                                          parse_method=parse_method)
 
         elif isinstance(sample, list):
             for id_, item in enumerate(sample):
                 k_ = f"{prefix}/{id_}"
-                cls._parse_helper(item, field_name, result_dic, prefix=k_, parse_method=parse_method)
+                cls._parse_helper(item, result_dic, field_name=field_name, prefix=k_, parse_method=parse_method)
diff --git a/dsdl/types/unstructure.py b/dsdl/types/unstructure.py
index aaf6d94..fd4365f 100644
--- a/dsdl/types/unstructure.py
+++ b/dsdl/types/unstructure.py
@@ -40,7 +40,7 @@ def validate(self, value):
 
 
 class LabelMapField(UnstructuredObjectField):
-    def __init__(self, dom, *args, **kwargs):
+    def __init__(self, dom=None, *args, **kwargs):
         super(LabelMapField, self).__init__(*args, **kwargs)
         self.dom = dom
 
diff --git a/setup.py b/setup.py
index 9ba9f89..05b71bf 100644
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,7 @@ def readme():
     author="DSDL team",
     author_email="dsdl-team@pjlab.org.cn",
     packages=find_packages(),
-    package_data={},
+    package_data={"dsdl": ["geometry/source/Arial_Font.ttf"]},
     include_package_data=True,
     install_requires=[
         "pyyaml>=6.0",


Test patch:
diff --git a/.github/workflows/test_pipeline.yml b/.github/workflows/test_pipeline.yml
index 3cb55d3..722cf44 100644
--- a/.github/workflows/test_pipeline.yml
+++ b/.github/workflows/test_pipeline.yml
@@ -1,10 +1,10 @@
 name: 'run test pipeline'
 
-on:
-  push:
-    branches: [ sherry_dev , dev-alg ]
-  pull_request:
-    branches: [ dev-alg ]
+#on:
+#  push:
+#    branches: [ sherry_dev , dev-alg ]
+#  pull_request:
+#    branches: [ dev-alg ]
 
 jobs:
   test: