Repository: Project-MONAI/MONAI
File: monai/data/dataset.py (Python)

Code before:
        _item_transformed = self._pre_transform(deepcopy(item_transformed))  # keep the original hashed
        if hashfile is None:
            return _item_transformed
        if isinstance(_item_transformed, dict):
            for k in _item_transformed:
                data_hashfile = f"{hashfile}-{k}"
                meta_hash_file_name = f"{hashfile.name}-{k}-meta"
                if isinstance(_item_transformed[k], (np.ndarray, torch.Tensor)):
                    self._create_new_cache(_item_transformed[k], data_hashfile, meta_hash_file_name)
                else:
                    return _item_transformed
        elif isinstance(_item_transformed, (np.ndarray, torch.Tensor)):
            data_hashfile = f"{hashfile}"
            meta_hash_file_name = f"{hashfile.name}-meta"
            self._create_new_cache(_item_transformed, data_hashfile, meta_hash_file_name)
        else:
            for i, _item in enumerate(_item_transformed):
                for k in _item:
                    data_hashfile = f"{hashfile}-{k}-{i}"
                    meta_hash_file_name = f"{hashfile.name}-{k}-meta-{i}"
                    self._create_new_cache(_item, data_hashfile, meta_hash_file_name)
        open(hashfile, "a").close()  # store cacheid
        return _item_transformed

    def _create_new_cache(self, data, data_hashfile, meta_hash_file_name):
        _item_transformed_meta = data.meta if isinstance(data, MetaTensor) else {}
        _item_transformed_data = data.array if isinstance(data, MetaTensor) else data
        if isinstance(_item_transformed_data, torch.Tensor):
            _item_transformed_data = _item_transformed_data.numpy()
        _item_transformed_meta["shape"] = _item_transformed_data.shape
        _item_transformed_meta["dtype"] = _item_transformed_data.dtype
        kvikio_numpy.tofile(_item_transformed_data, data_hashfile)
        try:
            # NOTE: Writing to a temporary directory and then using a nearly atomic rename operation
            #       to make the cache more robust to manual killing of parent process
            #       which may leave partially written cache files in an incomplete state
            with tempfile.TemporaryDirectory() as tmpdirname:
                meta_hash_file = self.cache_dir / meta_hash_file_name
                temp_hash_file = Path(tmpdirname) / meta_hash_file_name
                torch.save(
                    obj=_item_transformed_meta,
                    f=temp_hash_file,
                    pickle_module=look_up_option(self.pickle_module, SUPPORTED_PICKLE_MOD),
                    pickle_protocol=self.pickle_protocol,
                )
                if temp_hash_file.is_file() and not meta_hash_file.is_file():
                    # On Unix, if target exists and is a file, it will be replaced silently if the
                    # user has permission.
                    # for more details: https://docs.python.org/3/library/shutil.html#shutil.move.
                    try:
                        shutil.move(str(temp_hash_file), meta_hash_file)

Review comment (performance):
this meta dictionary doesn't take much space, how about storing it with `self._meta_cache["meta_hash_file_name"] = copy(data.meta)`?

Code after:
                    self._create_new_cache(_item_transformed[k], data_hashfile, meta_hash_file_name)
                else:
                    return _item_transformed
        elif isinstance(_item_transformed, (np.ndarray, torch.Tensor)):
            data_hashfile = f"{hashfile}"
            meta_hash_file_name = f"{hashfile.name}-meta"
            self._create_new_cache(_item_transformed, data_hashfile, meta_hash_file_name)
        else:
            for i, _item in enumerate(_item_transformed):
                for k in _item:
                    data_hashfile = f"{hashfile}-{k}-{i}"
                    meta_hash_file_name = f"{hashfile.name}-{k}-meta-{i}"
                    self._create_new_cache(_item, data_hashfile, meta_hash_file_name)
        open(hashfile, "a").close()  # store cacheid
        return _item_transformed

    def _create_new_cache(self, data, data_hashfile, meta_hash_file_name):
        self._meta_cache[meta_hash_file_name] = copy(data.meta) if isinstance(data, MetaTensor) else {}
        _item_transformed_data = data.array if isinstance(data, MetaTensor) else data
        if isinstance(_item_transformed_data, torch.Tensor):
            _item_transformed_data = _item_transformed_data.numpy()
        self._meta_cache[meta_hash_file_name]["shape"] = _item_transformed_data.shape
        self._meta_cache[meta_hash_file_name]["dtype"] = _item_transformed_data.dtype
        kvikio_numpy.tofile(_item_transformed_data, data_hashfile)
        try:
            # NOTE: Writing to a temporary directory and then using a nearly atomic rename operation
            #       to make the cache more robust to manual killing of parent process
            #       which may leave partially written cache files in an incomplete state
            with tempfile.TemporaryDirectory() as tmpdirname:
                meta_hash_file = self.cache_dir / meta_hash_file_name
                temp_hash_file = Path(tmpdirname) / meta_hash_file_name
                torch.save(
                    obj=self._meta_cache[meta_hash_file_name],
                    f=temp_hash_file,
                    pickle_module=look_up_option(self.pickle_module, SUPPORTED_PICKLE_MOD),
                    pickle_protocol=self.pickle_protocol,
                )
                if temp_hash_file.is_file() and not meta_hash_file.is_file():
                    # On Unix, if target exists and is a file, it will be replaced silently if the
                    # user has permission.
                    # for more details: https://docs.python.org/3/library/shutil.html#shutil.move.
                    try:
                        shutil.move(str(temp_hash_file), meta_hash_file)
                    except FileExistsError:
                        pass
        except PermissionError:  # project-monai/monai issue #3613
            pass

    def _load_meta_cache(self, meta_hash_file_name):
        if meta_hash_file_name in self._meta_cache:
            return self._meta_cache[meta_hash_file_name]