from typing import Dict, List, Tuple, TypedDict, Union import datasets as ds import evaluate import numpy as np import numpy.typing as npt _DESCRIPTION = """\ Some overlap metrics that are different to each other in previous works. """ _KWARGS_DESCRIPTION = """\ FIXME """ _CITATION = """\ @inproceedings{li2018layoutgan, title={LayoutGAN: Generating Graphic Layouts with Wireframe Discriminators}, author={Li, Jianan and Yang, Jimei and Hertzmann, Aaron and Zhang, Jianming and Xu, Tingfa}, booktitle={International Conference on Learning Representations}, year={2019} } @article{li2020attribute, title={Attribute-conditioned layout gan for automatic graphic design}, author={Li, Jianan and Yang, Jimei and Zhang, Jianming and Liu, Chang and Wang, Christina and Xu, Tingfa}, journal={IEEE Transactions on Visualization and Computer Graphics}, volume={27}, number={10}, pages={4039--4048}, year={2020}, publisher={IEEE} } @inproceedings{kikuchi2021constrained, title={Constrained graphic layout generation via latent optimization}, author={Kikuchi, Kotaro and Simo-Serra, Edgar and Otani, Mayu and Yamaguchi, Kota}, booktitle={Proceedings of the 29th ACM International Conference on Multimedia}, pages={88--96}, year={2021} } """ def convert_xywh_to_ltrb( batch_bbox: npt.NDArray[np.float64], ) -> Tuple[ npt.NDArray[np.float64], npt.NDArray[np.float64], npt.NDArray[np.float64], npt.NDArray[np.float64], ]: xc, yc, w, h = batch_bbox x1 = xc - w / 2 y1 = yc - h / 2 x2 = xc + w / 2 y2 = yc + h / 2 return (x1, y1, x2, y2) class A(TypedDict): a1: npt.NDArray[np.float64] ai: npt.NDArray[np.float64] class LayoutOverlap(evaluate.Metric): def _info(self) -> evaluate.EvaluationModuleInfo: return evaluate.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=ds.Features( { "bbox": ds.Sequence(ds.Sequence(ds.Value("float64"))), "mask": ds.Sequence(ds.Value("bool")), } ), codebase_urls=[ "https://github.com/ktrk115/const_layout/blob/master/metric.py#L138-L164", "https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L150-L203", ], ) def __calculate_a1_ai(self, batch_bbox: npt.NDArray[np.float64]) -> A: l1, t1, r1, b1 = convert_xywh_to_ltrb(batch_bbox[:, :, :, None]) l2, t2, r2, b2 = convert_xywh_to_ltrb(batch_bbox[:, :, None, :]) a1 = (r1 - l1) * (b1 - t1) # shape: (B, S, S) l_max = np.maximum(l1, l2) r_min = np.minimum(r1, r2) t_max = np.maximum(t1, t2) b_min = np.minimum(b1, b2) cond = (l_max < r_min) & (t_max < b_min) ai = np.where(cond, (r_min - l_max) * (b_min - t_max), 0.0) return {"a1": a1, "ai": ai} def _compute_ac_layout_gan( self, S: int, ai: npt.NDArray[np.float64], a1: npt.NDArray[np.float64], batch_mask: npt.NDArray[np.bool_], ) -> npt.NDArray[np.float64]: # shape: (B, S) -> (B, S, S) batch_mask = ~batch_mask[:, None, :] | ~batch_mask[:, :, None] indices = np.arange(S) batch_mask[:, indices, indices] = True ai[batch_mask] = 0.0 # shape: (B, S, S) ar = np.nan_to_num(ai / a1) score = ar.sum(axis=(1, 2)) return score def _compute_layout_gan_pp( self, score_ac_layout_gan: npt.NDArray[np.float64], batch_mask: npt.NDArray[np.bool_], ) -> npt.NDArray[np.float64]: # shape: (B, S) -> (B,) batch_mask = batch_mask.sum(axis=1) # shape: (B,) score_normalized = score_ac_layout_gan / batch_mask score_normalized[np.isnan(score_normalized)] = 0.0 return score_normalized def _compute_layout_gan( self, S: int, B: int, ai: npt.NDArray[np.float64] ) -> npt.NDArray[np.float64]: indices = np.arange(S) ii, jj = np.meshgrid(indices, indices, indexing="ij") # shape: ii (S, S) -> (1, S, S), jj (S, S) -> (1, S, S) # shape: (1, S, S) -> (B, S, S) ai[np.repeat((ii[None, :] >= jj[None, :]), axis=0, repeats=B)] = 0.0 # shape: (B, S, S) -> (B,) score = ai.sum(axis=(1, 2)) return score def _compute( self, *, bbox: Union[npt.NDArray[np.float64], List[List[int]]], mask: Union[npt.NDArray[np.bool_], List[List[bool]]], ) -> Dict[str, npt.NDArray[np.float64]]: # shape: (B, model_max_length, C) bbox = np.array(bbox) # shape: (B, model_max_length) mask = np.array(mask) assert bbox.ndim == 3 assert mask.ndim == 2 # S: model_max_length B, S, C = bbox.shape # shape: batch_bbox (B, S, C), batch_mask (B, S) -> (B, S, 1) -> (B, S, C) bbox[np.repeat(~mask[:, :, None], axis=2, repeats=C)] = 0.0 # shape: (C, B, S) bbox = bbox.transpose(2, 0, 1) A = self.__calculate_a1_ai(bbox) # shape: (B,) score_ac_layout_gan = self._compute_ac_layout_gan(S=S, batch_mask=mask, **A) # shape: (B,) score_layout_gan_pp = self._compute_layout_gan_pp( score_ac_layout_gan=score_ac_layout_gan, batch_mask=mask ) # shape: (B,) score_layout_gan = self._compute_layout_gan(B=B, S=S, ai=A["ai"]) return { "overlap-ACLayoutGAN": score_ac_layout_gan, "overlap-LayoutGAN++": score_layout_gan_pp, "overlap-LayoutGAN": score_layout_gan, }