initial commit

.gitignore

@@ -0,0 +1,24 @@
+# block temp directory
+.idea/
+__pycache__/
+.ipynb_checkpoints/
+.vscode/
+.temp/
+lightning_logs/
+
+# block extension
+*.pkl
+*.png
+*.pth
+*.json
+*.ckpt
+
+# block logging directory
+logs/
+wandb/
+
+# custom
+font-image
+
+!font_list.json
+!font_list_noto_sans.json

app.py

@@ -0,0 +1,121 @@
+import os
+import argparse
+from pathlib import Path
+from typing import Optional, Union, Tuple, List
+import subprocess
+
+import gradio as gr
+from PIL import Image
+from omegaconf import OmegaConf, DictConfig
+
+from inference import InferenceServicer
+
+PATH_DOCS = os.getenv("PATH_DOCS", default="docs/ml-font-style-transfer.md")
+MODEL_CONFIG = os.getenv("MODEL_CONFIG", default="config/models/google-font.yaml")
+
+MODEL_CHECKPOINT_PATH = os.getenv("MODEL_CHECKPOINT_PATH", default=None)
+NOTO_SANS_ZIP_PATH = os.getenv("NOTO_SANS_ZIP_PATH", default=None)
+
+LOCAL_CHECKPOINT_PATH = "checkpoint/checkpoint.ckpt"
+LOCAL_NOTO_ZIP_PATH = "data/NotoSans.zip"
+
+if MODEL_CHECKPOINT_PATH is not None:
+    subprocess.call(f"wget --no-check-certificate -O {LOCAL_CHECKPOINT_PATH} {MODEL_CHECKPOINT_PATH}", shell=True)
+if NOTO_SANS_ZIP_PATH is not None:
+    subprocess.call(f"wget --no-check-certificate -O {LOCAL_NOTO_ZIP_PATH} {NOTO_SANS_ZIP_PATH}", shell=True)
+subprocess.call(f"unzip data/NotoSans.zip -d {str(Path(LOCAL_NOTO_ZIP_PATH).parent)}", shell=True)
+
+assert Path("checkpoint/checkpoint.ckpt").exists()
+assert Path("data/NotoSans").exists()
+
+EXAMPLE_FONTS = sorted([
+    "example_fonts/BalooDa2-Bold.ttf",
+    "example_fonts/BalooDa2-Regular.ttf",
+    "example_fonts/Lalezar-Regular.ttf",
+    "example_fonts/MaShanZheng-Regular.ttf",
+])
+
+def parse_args():
+
+    parser = argparse.ArgumentParser(description="Augmentation simulator for NetsPresso Trainer")
+
+    # -------- User arguments ----------------------------------------
+
+    parser.add_argument(
+        '--docs', type=Path, default=PATH_DOCS,
+        help="Docs string file")
+
+    parser.add_argument(
+        '--config', type=Path, default=MODEL_CONFIG,
+        help="Config for model")
+
+    parser.add_argument(
+        '--local', action='store_true',
+        help="Whether to run in local environment or not")
+
+    parser.add_argument(
+        '--port', type=int, default=50003,
+        help="Service port (only applicable when running on local server)")
+
+    args, _ = parser.parse_known_args()
+
+    return args
+
+class InferenceServiceResolver(InferenceServicer):
+    def __init__(self, hp, checkpoint_path, content_image_dir, imsize=64, gpu_id='0') -> None:
+        super().__init__(hp, checkpoint_path, content_image_dir, imsize, gpu_id)
+        
+    def generate(self, content_char: str, style_font: Union[str, Path]) -> List[Image.Image]:
+        try:
+            content_image, style_images, result = self.inference(content_char=content_char, style_font=style_font)
+            return [content_image, *style_images, result]
+        except Exception as e:
+            raise gr.Error(str(e))
+
+def launch_gradio(docs_path: Path, hp: DictConfig, checkpoint_path: Path, content_image_dir: Path, is_local: bool, port: Optional[int] = None):
+    
+    servicer = InferenceServiceResolver(hp, checkpoint_path, content_image_dir, gpu_id=None)
+    with gr.Blocks(title="Multilingual Font Style Transfer (training with Google Fonts)") as demo:
+        gr.Markdown(docs_path.read_text())
+        with gr.Row(equal_height=True):
+            character_input = gr.Textbox(max_lines=1, value="7", info="Only single character is acceptable (e.g. '간', '7', or 'ជ')")
+            style_font = gr.Dropdown(label="Select example font: ", choices=EXAMPLE_FONTS, value=EXAMPLE_FONTS[0])
+            run_button = gr.Button(value="Generate", variant='primary')
+        
+        with gr.Row(equal_height=True):
+            with gr.Column(scale=1):
+                with gr.Group():
+                    gr.Markdown(f"<center><h3>Content character</h3></center>")
+                    content_char = gr.Image(label="Content character", show_label=False)
+            with gr.Column(scale=5):
+                with gr.Group():
+                    gr.Markdown(f"<center><h3>Style font images</h3></center>")
+                    with gr.Row(equal_height=True):
+                        style_char_1 = gr.Image(label="Style #1", show_label=False)
+                        style_char_2 = gr.Image(label="Style #2", show_label=False)
+                        style_char_3 = gr.Image(label="Style #3", show_label=False)
+                        style_char_4 = gr.Image(label="Style #4", show_label=False)
+                        style_char_5 = gr.Image(label="Style #5", show_label=False)
+            with gr.Column(scale=1):
+                with gr.Group():
+                    gr.Markdown(f"<center><h3>Generated font image</h3></center>")
+                    generated_font = gr.Image(label="Generated font image", show_label=False)
+
+        outputs = [content_char, style_char_1, style_char_2, style_char_3, style_char_4, style_char_5, generated_font]
+        run_inputs = [character_input, style_font]
+        run_button.click(servicer.generate, inputs=run_inputs, outputs=outputs)
+
+    if is_local:
+        demo.launch(server_name="0.0.0.0", server_port=port)
+    else:
+        demo.launch()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    
+    hp = OmegaConf.load(args.config)
+    checkpoint_path = Path(LOCAL_CHECKPOINT_PATH)
+    content_image_dir = Path(LOCAL_NOTO_ZIP_PATH).with_suffix("")
+    
+    launch_gradio(args.docs, hp, checkpoint_path, content_image_dir, args.local, args.port)

config/datasets/googlefont.yaml

@@ -0,0 +1,37 @@
+datasets:
+  type: GoogleFontDataset
+  train:
+    split: auto
+    font_dir: &font_dir ../DATA/fonts-image-20230929
+    imsize: 64
+    reference_imgs:
+      replace: False
+      char: &reference_char 1
+      style: &reference_style 5
+
+    squeeze_gray: &squeeze_gray True
+    transform:
+    # TODO
+
+    # loader configs
+    shuffle: True
+    batch_size: 64
+    num_workers: 12
+
+  eval:
+    split: auto
+    font_dir: *font_dir
+    imsize: 64
+    reference_imgs:
+      replace: False
+      char: *reference_char
+      style: *reference_style
+
+    squeeze_gray: *squeeze_gray
+    transform:
+    # TODO
+
+    # loader configs
+    shuffle: True
+    batch_size: 1
+    num_workers: 4

config/lightning.yaml

@@ -0,0 +1,16 @@
+pl_config:
+  checkpoint:
+    callback:
+      save_top_k: -1
+      verbose: True
+      every_n_epochs: 5 #epochs
+
+  trainer:
+    gradient_clip_val: 0
+    max_epochs: 2000
+    num_sanity_val_steps: 1
+    fast_dev_run: False
+    check_val_every_n_epoch: 5
+    # distributed_backend: 'ddp'
+    accelerator: 'cuda'
+    benchmark: True

config/logging.yaml

@@ -0,0 +1,25 @@
+logging:
+  dry_run: False
+  device: cuda
+  log_dir: /ssd1/hksong/LOG/font
+  seed: ftgan-patch-full
+  freq:
+    train: 100 # step
+
+  nepochs_decay: 100
+
+  gan_loss: lsgan
+  lambda_L1: 100
+  lambda_classifier: ~
+
+  trainer: base
+
+  savefiles: [
+      '*.py',
+      'data/*.*',
+      'datasets/*.*',
+      'models/*.*',
+      'configs/*.*',
+      'utils/*.*',
+      'trainer/*.*',
+  ]

config/models/google-font.yaml

@@ -0,0 +1,43 @@
+models:
+  G:
+    encoder:
+      content:
+        type: ContentVanillaEncoder
+        depth: 2
+      style:
+        type: StyleVanillaEncoder
+        depth: 2
+    decoder:
+      type: VanillaDecoder
+      residual_blocks: 6
+      depth: 2
+    
+    optim:
+      class: torch.optim.Adam
+      betas: [ 0.5, 0.999 ]
+      lr: 0.0002
+      lr_policy: step
+      lr_decay_iters: 1000
+
+    init_type: normal
+    init_gain: 0.02
+
+  D_content:
+    in_channels: 2 # char + 1
+    class: models.discriminator.PatchGANDiscriminator
+    optim:
+      class: torch.optim.Adam
+      betas: [ 0.5, 0.999 ]
+      lr: 2e-4
+      lr_policy: step
+      lr_decay_iters: 1000
+
+  D_style:
+    in_channels: 6 # style + 1
+    class: models.discriminator.PatchGANDiscriminator
+    optim:
+      class: torch.optim.Adam
+      betas: [ 0.5, 0.999 ]
+      lr: 2e-4
+      lr_policy: step
+      lr_decay_iters: 1000

config/setting-google-font.yaml

@@ -0,0 +1,5 @@
+config:
+  dataset: 'config/datasets/googlefont.yaml'
+  model: 'config/models/google-font.yaml'
+  logging: 'config/logging.yaml'
+  lightning: 'config/lightning.yaml'

datasets/__init__.py

@@ -0,0 +1,2 @@
+from .googlefont import GoogleFontDataset
+from .ftgan import FTGANDataset

datasets/googlefont.py

@@ -0,0 +1,286 @@
+import os
+import pickle
+import random
+import string
+import json
+import logging
+from pathlib import Path
+
+from omegaconf import OmegaConf
+import numpy as np
+import PIL.Image as Image
+import torch
+from torch.utils.data import Dataset
+from tqdm import tqdm
+
+REPEATE_NUM = 10000
+
+WHITE = 255
+
+MAX_TRIAL = 10
+
+_upper_case = set(map(lambda s: f"{ord(s):04X}", string.ascii_uppercase))
+_digits = set(map(lambda s: f"{ord(s):04X}", string.digits))
+english_set = list(_upper_case.union(_digits))
+
+NOTO_FONT_DIRNAME = "Noto"
+
+
+class GoogleFontDataset(Dataset):
+    def __init__(self, args, mode='train',
+                 metadata_path="./lang_set.json"):
+        super(GoogleFontDataset, self).__init__()
+        self.args = args
+        self.font_dir = Path(args.font_dir)
+        self.mode = mode
+        self.lang_list = sorted([x.stem for x in self.font_dir.iterdir() if x.is_dir()])
+        self.min_tight_bound = 10000
+        self.min_font_name = None
+
+        if self.mode == 'train':
+            self.lang_list = self.lang_list[:-2]
+        else:
+            self.lang_list = self.lang_list[-2:]
+        with open(metadata_path, "r") as json_f:
+            self.data = json.load(json_f)
+
+        self.num_lang = None
+        self.num_font = None
+        self.num_char = None
+        self.content_meta, self.style_meta, self.num_lang, self.num_font, self.num_char = self.get_meta()
+        logging.info(f"min_tight_bound: {self.min_tight_bound}")  # 20
+
+    @staticmethod
+    def center_align(bg_img, item_img, fit=False):
+        bg_img = bg_img.copy()
+        item_img = item_img.copy()
+        item_w, item_h = item_img.size
+        W, H = bg_img.size
+        if fit:
+            item_ratio = item_w / item_h
+            bg_ratio = W / H
+
+            if bg_ratio > item_ratio:
+                # height fitting
+                resize_ratio = H / item_h
+            else:
+                # width fitting
+                resize_ratio = W / item_w
+            item_img = item_img.resize((int(item_w * resize_ratio), int(item_h * resize_ratio)))
+            item_w, item_h = item_img.size
+
+        bg_img.paste(item_img, ((W - item_w) // 2, (H - item_h) // 2))
+        return bg_img
+
+    def _get_content_image(self, png_path):
+        im = Image.open(png_path)
+        bg_img = Image.new('RGB', (self.args.imsize, self.args.imsize), color='white')
+        blend_img = self.center_align(bg_img, im, fit=True)
+        return blend_img
+
+    def _get_style_image(self, png_path):
+        im = Image.open(png_path)
+        w, h = im.size
+
+        # tight_bound_check & update
+        tight_bound = self.get_tight_bound_size(np.array(im))
+        if self.min_tight_bound > tight_bound:
+            self.min_tight_bound = tight_bound
+            self.min_font_name = png_path
+            logging.debug(f"min_tight_bound: {self.min_tight_bound}, min_font_name: {self.min_font_name}")
+
+        bg_img = Image.new('RGB', (max([w, h, self.args.imsize]), max([w, h, self.args.imsize])), color='white')
+        blend_img = self.center_align(bg_img, im)
+        return blend_img
+
+    def get_meta(self):
+        content_meta = dict()
+        style_meta = dict()
+
+        num_lang = 0
+        num_font = 0
+        num_char = 0
+        for lang_dir in tqdm(self.lang_list, total=len(self.lang_list)):
+            font_list = sorted([x for x in (self.font_dir / lang_dir).iterdir() if x.is_dir()])
+
+            font_content_dict = dict()
+            font_style_dict = dict()
+
+            for font_dir in font_list:
+                image_content_dict = dict()
+                image_style_dict = dict()
+
+                png_list = [x for x in font_dir.glob("*.png")]
+
+                for png_path in png_list:
+
+                    # image_content_dict[png_path.stem] = self._get_content_image(png_path)
+                    # image_style_dict[png_path.stem] = self._get_style_image(png_path)
+                    image_content_dict[png_path.stem] = png_path
+                    image_style_dict[png_path.stem] = png_path
+                    num_char += 1
+
+                font_content_dict[font_dir.stem] = image_content_dict
+                font_style_dict[font_dir.stem] = image_style_dict
+                num_font += 1
+
+            content_meta[lang_dir] = font_content_dict
+            style_meta[lang_dir] = font_style_dict
+            num_lang += 1
+
+        return content_meta, style_meta, num_lang, num_font, num_char
+
+    @staticmethod
+    def get_tight_bound_size(img):
+        contents_cell = np.where(img < WHITE)
+
+        if len(contents_cell[0]) == 0:
+            return 0
+
+        size = {
+            'xmin': np.min(contents_cell[1]),
+            'ymin': np.min(contents_cell[0]),
+            'xmax': np.max(contents_cell[1]) + 1,
+            'ymax': np.max(contents_cell[0]) + 1,
+        }
+        return max(size['xmax'] - size['xmin'], size['ymax'] - size['ymin'])
+
+    def get_patch_from_style_image(self, image, patch_per_image=1):
+        w, h = image.size
+        image_list = []
+        relative_patch_size = int(self.args.imsize * 2)
+        for _ in range(patch_per_image):
+            offset = w - relative_patch_size
+            if offset < relative_patch_size // 2:
+                # if image is too small, just resize
+                crop_candidate = np.array(image.resize((self.args.imsize, self.args.imsize)))
+            else:
+                # if image is sufficent to be cropped, randomly crop
+                x = np.random.randint(0, offset)
+                y = np.random.randint(0, offset)
+                crop_candidate = image.crop((x, y, x + relative_patch_size, y + relative_patch_size))
+
+                _trial = 0
+                while self.get_tight_bound_size(np.array(crop_candidate)) < relative_patch_size // 16 and _trial < MAX_TRIAL:
+                    x = np.random.randint(0, offset)
+                    y = np.random.randint(0, offset)
+                    crop_candidate = image.crop((x, y, x + relative_patch_size, y + relative_patch_size))
+                    _trial += 1
+
+                crop_candidate = np.array(crop_candidate.resize((self.args.imsize, self.args.imsize)))
+            image_list.append(crop_candidate)
+        return image_list
+
+    def get_pairs(self, content_english=False, style_english=False):
+        lang_content = random.choice(self.lang_list)
+
+        content_unicode_list = english_set if content_english else self.data[lang_content]
+        style_unicode_list = english_set if style_english else self.data[lang_content]
+
+        if content_english == style_english:
+            # content_unicode_list == style_unicode_list
+            chars = random.sample(content_unicode_list,
+                                  k=self.args.reference_imgs.style + 1)
+            content_char = chars[-1]
+            style_chars = chars[:self.args.reference_imgs.style]
+        else:
+            content_char = random.choice(content_unicode_list)
+            style_chars = random.sample(style_unicode_list, k=self.args.reference_imgs.style)
+
+        # fonts = random.sample(self.content_meta[lang_content].keys(),
+        #                       k=self.args.reference_imgs.char + 1)
+        # content_fonts = fonts[:self.args.reference_imgs.char]
+        # style_font = fonts[-1]
+        
+        style_font_list = list(self.content_meta[lang_content].keys())
+        style_font_list.remove(NOTO_FONT_DIRNAME)
+        style_font = random.choice(style_font_list)
+        content_fonts = [NOTO_FONT_DIRNAME]
+
+        content_fonts_image = [self.content_meta[lang_content][x][content_char] for x in content_fonts]
+        style_chars_image = [self.content_meta[lang_content][style_font][x] for x in style_chars]
+
+        # style_chars_image = [self.content_meta[lang_content][style_font][x] for x in style_chars]
+
+        # style_chars_cropped = []
+        # for style_char_image in style_chars_image:
+        #     style_chars_cropped.extend(self.get_patch_from_style_image(style_char_image,
+        #                                                                patch_per_image=self.args.reference_imgs.style // self.args.reference_imgs.char))
+
+        target_image = self.content_meta[lang_content][style_font][content_char]
+        
+        content_fonts_image = [self._get_content_image(image_path) for image_path in content_fonts_image]
+        style_chars_image = [self._get_content_image(image_path) for image_path in style_chars_image]
+        target_image = self._get_content_image(target_image)
+
+        return content_char, content_fonts, content_fonts_image, style_font, style_chars, style_chars_image, target_image
+
+    def __getitem__(self, idx):
+        """GoogleFontDataset의 __getitem__
+
+        Args:
+            idx (int): torch dataset index
+
+        Returns:
+            dict: return dict with following keys
+
+            gt_images: target_image,
+            content_images: same_chars_image,
+            style_images: same_fonts_image,
+            style_idx: font_idx,
+            char_idx: char_idx,
+            content_image_idxs: same_chars,
+            style_image_idxs: same_fonts,
+            image_paths: ''
+        """
+        use_eng_content, use_eng_style = random.choice([(True, False), (False, True), (False, False)])
+
+        if self.mode != 'train':
+            use_eng_content = False
+            use_eng_style = True
+
+        content_char, content_fonts, content_fonts_image, style_font, style_chars, style_chars_image, target_image = \
+            self.get_pairs(content_english=use_eng_content, style_english=use_eng_style)
+
+        content_fonts_image = np.array([np.mean(np.array(x), axis=-1) / WHITE
+                                        for x in content_fonts_image], dtype=np.float32)
+        style_chars_image = np.array([np.mean(np.array(x), axis=-1) / WHITE
+                                      for x in style_chars_image], dtype=np.float32)
+        target_image = np.mean(np.array(target_image,  dtype=np.float32), axis=-1)[np.newaxis, ...] / WHITE
+
+        dict_return = {
+            # data for training
+            'gt_images': target_image,
+            'content_images': content_fonts_image,
+            'style_images': style_chars_image,  # TODO: crop style image with fixed size
+            # data for logging
+            'style_idx': style_font,
+            'char_idx': content_char,
+            'content_image_idxs': content_fonts,
+            'style_image_idxs': style_chars,
+            'image_paths': '',
+        }
+        return dict_return
+
+    def __len__(self):
+        return len(self.lang_list) * REPEATE_NUM
+
+
+if __name__ == '__main__':
+    hp = OmegaConf.load('config/datasets/googlefont.yaml').datasets.train
+    metadata_path = "./lang_set.json"
+    FONT_DIR = "/data2/hksong/DATA/fonts-image"
+
+    _dataset = GoogleFontDataset(hp, metadata_path=metadata_path, font_dir=FONT_DIR)
+    TEST_ITER_NUM = 4
+    for i in range(TEST_ITER_NUM):
+        data = _dataset[i]
+        print(data.keys())
+        print(data['gt_image'].size,
+              data['content_images'][0].size,
+              data['style_images'][0].size,
+              data['lang'],
+              data['style_idx'],
+              data['char_idx'],
+              data['content_image_idxs'],
+              data['style_image_idxs'])

docs/ml-font-style-transfer.md

@@ -0,0 +1,9 @@
+<center><h1>Multilingual Font Style Transfer</h1></center>
+
+- Compostion-free font style transfer across 13 different languages
+- Trained with [Google Fonts](https://github.com/google/fonts) (ofl fonts and Nota Sans)
+
+This is personal concept proofing demo, so it does not guarantee that the quality of output. 
+I hope that in someday there will be an established model for the better mulitlingual society.  
+
+I only used personal RTX 30 series GPU(s) for training the model. The model is heavily inspired from a model from the previous study, [FTransGAN](https://github.com/ligoudaner377/font_translator_gan) (Li et al.).

font_list.json

@@ -0,0 +1,427 @@
+{
+    "arabic": [
+        "ofl/baloobhaijaan2/BalooBhaijaan2[wght].ttf",
+        "ofl/bonanova/BonaNova-Italic.ttf",
+        "ofl/bonanova/BonaNova-Regular.ttf",
+        "ofl/cairo/Cairo[wght].ttf",
+        "ofl/changa/Changa[wght].ttf",
+        "ofl/elmessiri/ElMessiri[wght].ttf",
+        "ofl/handjet/Handjet[EGRD,ESHP,wght].ttf",
+        "ofl/harmattan/Harmattan-Regular.ttf",
+        "ofl/ibmplexsansarabic/IBMPlexSansArabic-Regular.ttf",
+        "ofl/katibeh/Katibeh-Regular.ttf",
+        "ofl/kufam/Kufam-Italic[wght].ttf",
+        "ofl/kufam/Kufam[wght].ttf",
+        "ofl/lalezar/Lalezar-Regular.ttf",
+        "ofl/lemonada/Lemonada[wght].ttf",
+        "ofl/lemonadavfbeta/LemonadaVFBeta.ttf",
+        "ofl/markazitext/MarkaziText[wght].ttf",
+        "ofl/mirza/Mirza-Regular.ttf",
+        "ofl/qahiri/Qahiri-Regular.ttf",
+        "ofl/rakkas/Rakkas-Regular.ttf",
+        "ofl/readexpro/ReadexPro[wght].ttf",
+        "ofl/reemkufi/ReemKufi[wght].ttf",
+        "ofl/scheherazadenew/ScheherazadeNew-Regular.ttf",
+        "ofl/scheherazade/Scheherazade-Regular.ttf",
+        "ofl/tajawal/Tajawal-Regular.ttf",
+        "ofl/vibes/Vibes-Regular.ttf"
+    ],
+    "bengali": [
+        "ofl/atma/Atma-Regular.ttf",
+        "ofl/balooda2/BalooDa2-Regular.ttf",
+        "ofl/galada/Galada-Regular.ttf",
+        "ofl/hindsiliguri/HindSiliguri-Regular.ttf",
+        "ofl/mina/Mina-Regular.ttf"
+    ],
+    "gujarati": [
+        "ofl/baloobhai2/BalooBhai2[wght].ttf",
+        "ofl/farsan/Farsan-Regular.ttf",
+        "ofl/hindvadodara/HindVadodara-Regular.ttf",
+        "ofl/mogra/Mogra-Regular.ttf",
+        "ofl/muktavaani/MuktaVaani-Regular.ttf",
+        "ofl/rasa/Rasa[wght].ttf",
+        "ofl/shrikhand/Shrikhand-Regular.ttf"
+    ],
+    "hebrew": [
+        "ofl/adobeblank/AdobeBlank-Regular.ttf",
+        "ofl/alef/Alef-Regular.ttf",
+        "ofl/amaticsc/AmaticSC-Regular.ttf",
+        "ofl/bellefair/Bellefair-Regular.ttf",
+        "ofl/bonanova/BonaNova-Italic.ttf",
+        "ofl/bonanova/BonaNova-Regular.ttf",
+        "ofl/cardo/Cardo-Italic.ttf",
+        "ofl/cardo/Cardo-Regular.ttf",
+        "ofl/davidlibre/DavidLibre-Regular.ttf",
+        "ofl/frankruhllibre/FrankRuhlLibre-Regular.ttf",
+        "ofl/handjet/Handjet[EGRD,ESHP,wght].ttf",
+        "ofl/heebo/Heebo[wght].ttf",
+        "ofl/ibmplexsanshebrew/IBMPlexSansHebrew-Regular.ttf",
+        "ofl/karantina/Karantina-Regular.ttf",
+        "ofl/miriamlibre/MiriamLibre-Regular.ttf",
+        "ofl/mplus1p/Mplus1p-Regular.ttf",
+        "ofl/roundedmplus1c/RoundedMplus1c-Regular.ttf",
+        "ofl/rubikbeastly/RubikBeastly-Regular.ttf",
+        "ofl/rubikmonoone/RubikMonoOne-Regular.ttf",
+        "ofl/rubikone/RubikOne-Regular.ttf",
+        "ofl/rubik/Rubik-Italic[wght].ttf",
+        "ofl/secularone/SecularOne-Regular.ttf",
+        "ofl/suezone/SuezOne-Regular.ttf"
+    ],
+    "japanese": [
+        "ofl/delagothicone/DelaGothicOne-Regular.ttf",
+        "ofl/dotgothic16/DotGothic16-Regular.ttf",
+        "ofl/hachimarupop/HachiMaruPop-Regular.ttf",
+        "ofl/jejugothic/JejuGothic-Regular.ttf",
+        "ofl/jejuhallasan/JejuHallasan-Regular.ttf",
+        "ofl/jejumyeongjo/JejuMyeongjo-Regular.ttf",
+        "ofl/kaiseidecol/KaiseiDecol-Regular.ttf",
+        "ofl/kaiseiharunoumi/KaiseiHarunoUmi-Regular.ttf",
+        "ofl/kaiseiopti/KaiseiOpti-Regular.ttf",
+        "ofl/kaiseitokumin/KaiseiTokumin-Regular.ttf"
+    ],
+    "khmer": [
+        "ofl/angkor/Angkor-Regular.ttf",
+        "ofl/battambang/Battambang-Regular.ttf",
+        "ofl/bayon/Bayon-Regular.ttf",
+        "ofl/bokor/Bokor-Regular.ttf",
+        "ofl/dangrek/Dangrek-Regular.ttf",
+        "ofl/fasthand/Fasthand-Regular.ttf",
+        "ofl/freehand/Freehand-Regular.ttf",
+        "ofl/hanuman/Hanuman-Regular.ttf",
+        "ofl/kohsantepheap/KohSantepheap-Regular.ttf",
+        "ofl/koulen/Koulen-Regular.ttf",
+        "ofl/metal/Metal-Regular.ttf",
+        "ofl/moul/Moul-Regular.ttf",
+        "ofl/moulpali/Moulpali-Regular.ttf",
+        "ofl/nokora/Nokora-Regular.ttf",
+        "ofl/odormeanchey/OdorMeanChey-Regular.ttf",
+        "ofl/preahvihear/Preahvihear-Regular.ttf",
+        "ofl/suwannaphum/Suwannaphum-Regular.ttf",
+        "ofl/taprom/Taprom-Regular.ttf"
+    ],
+    "korean": [
+        "ofl/blackandwhitepicture/BlackAndWhitePicture-Regular.ttf",
+        "ofl/dongle/Dongle-Regular.ttf",
+        "ofl/gamjaflower/GamjaFlower-Regular.ttf",
+        "ofl/gothica1/GothicA1-Regular.ttf",
+        "ofl/gowunbatang/GowunBatang-Regular.ttf",
+        "ofl/gowundodum/GowunDodum-Regular.ttf",
+        "ofl/himelody/HiMelody-Regular.ttf",
+        "ofl/poorstory/PoorStory-Regular.ttf"
+    ],
+    "malayalam": [
+        "ofl/baloochettan2/BalooChettan2-Regular.ttf",
+        "ofl/chilanka/Chilanka-Regular.ttf",
+        "ofl/gayathri/Gayathri-Regular.ttf",
+        "ofl/hindkochi/HindKochi-Regular.ttf",
+        "ofl/manjari/Manjari-Regular.ttf"
+    ],
+    "cyrillic": [
+        "ofl/adobeblank/AdobeBlank-Regular.ttf",
+        "ofl/alegreya/Alegreya-Italic[wght].ttf",
+        "ofl/alegreya/Alegreya[wght].ttf",
+        "ofl/alegreyasans/AlegreyaSans-Italic.ttf",
+        "ofl/alegreyasans/AlegreyaSans-Regular.ttf",
+        "ofl/alegreyasanssc/AlegreyaSansSC-Italic.ttf",
+        "ofl/alegreyasanssc/AlegreyaSansSC-Regular.ttf",
+        "ofl/alegreyasc/AlegreyaSC-Italic.ttf",
+        "ofl/alegreyasc/AlegreyaSC-Regular.ttf",
+        "ofl/alice/Alice-Regular.ttf",
+        "ofl/alumnisans/AlumniSans-Italic[wght].ttf",
+        "ofl/amaticsc/AmaticSC-Regular.ttf",
+        "ofl/andika/Andika-Regular.ttf",
+        "ofl/anonymouspro/AnonymousPro-Italic.ttf",
+        "ofl/anonymouspro/AnonymousPro-Regular.ttf",
+        "ofl/arsenal/Arsenal-Italic.ttf",
+        "ofl/arsenal/Arsenal-Regular.ttf",
+        "ofl/badscript/BadScript-Regular.ttf",
+        "ofl/balsamiqsans/BalsamiqSans-Italic.ttf",
+        "ofl/balsamiqsans/BalsamiqSans-Regular.ttf",
+        "ofl/bellota/Bellota-Italic.ttf",
+        "ofl/bellota/Bellota-Regular.ttf",
+        "ofl/bellotatext/BellotaText-Italic.ttf",
+        "ofl/bellotatext/BellotaText-Regular.ttf",
+        "ofl/bitter/Bitter-Italic[wght].ttf",
+        "ofl/bonanova/BonaNova-Italic.ttf",
+        "ofl/bonanova/BonaNova-Regular.ttf",
+        "ofl/brygada1918/Brygada1918-Italic[wght].ttf",
+        "ofl/brygada1918/Brygada1918[wght].ttf",
+        "ofl/caveat/Caveat[wght].ttf",
+        "ofl/comfortaa/Comfortaa[wght].ttf",
+        "ofl/comforterbrush/ComforterBrush-Regular.ttf",
+        "ofl/comforter/Comforter-Regular.ttf",
+        "ofl/cormorant/Cormorant-Italic.ttf",
+        "ofl/cormorant/Cormorant-Regular.ttf",
+        "ofl/cormorantgaramond/CormorantGaramond-Italic.ttf",
+        "ofl/cormorantgaramond/CormorantGaramond-Regular.ttf",
+        "ofl/cormorantinfant/CormorantInfant-Italic.ttf",
+        "ofl/cormorantinfant/CormorantInfant-Regular.ttf",
+        "ofl/cormorantsc/CormorantSC-Regular.ttf",
+        "ofl/cormorantunicase/CormorantUnicase-Regular.ttf",
+        "ofl/crimsontext/CrimsonText-Regular.ttf",
+        "ofl/cuprum/Cuprum-Italic[wght].ttf",
+        "ofl/cuprum/Cuprum[wght].ttf",
+        "ofl/daysone/DaysOne-Regular.ttf",
+        "ofl/delagothicone/DelaGothicOne-Regular.ttf",
+        "ofl/didactgothic/DidactGothic-Regular.ttf",
+        "ofl/dotgothic16/DotGothic16-Regular.ttf",
+        "ofl/ebgaramond/EBGaramond-Italic[wght].ttf",
+        "ofl/ebgaramond/EBGaramond[wght].ttf",
+        "ofl/elmessiri/ElMessiri[wght].ttf",
+        "ofl/exo2/Exo2-Italic[wght].ttf",
+        "ofl/exo2/Exo2[wght].ttf",
+        "ofl/firasanscondensed/FiraSansCondensed-Italic.ttf",
+        "ofl/firasanscondensed/FiraSansCondensed-Regular.ttf",
+        "ofl/firasansextracondensed/FiraSansExtraCondensed-Italic.ttf",
+        "ofl/firasansextracondensed/FiraSansExtraCondensed-Regular.ttf",
+        "ofl/firasans/FiraSans-Italic.ttf",
+        "ofl/firasans/FiraSans-Regular.ttf",
+        "ofl/flowblock/FlowBlock-Regular.ttf",
+        "ofl/flowcircular/FlowCircular-Regular.ttf",
+        "ofl/flowrounded/FlowRounded-Regular.ttf",
+        "ofl/forum/Forum-Regular.ttf",
+        "ofl/gabriela/Gabriela-Regular.ttf",
+        "ofl/gothica1/GothicA1-Regular.ttf",
+        "ofl/hachimarupop/HachiMaruPop-Regular.ttf",
+        "ofl/handjet/Handjet[EGRD,ESHP,wght].ttf",
+        "ofl/hinamincho/HinaMincho-Regular.ttf",
+        "ofl/ibmplexmono/IBMPlexMono-Italic.ttf",
+        "ofl/ibmplexmono/IBMPlexMono-Regular.ttf",
+        "ofl/ibmplexsans/IBMPlexSans-Italic.ttf",
+        "ofl/ibmplexsans/IBMPlexSans-Regular.ttf",
+        "ofl/ibmplexserif/IBMPlexSerif-Italic.ttf",
+        "ofl/ibmplexserif/IBMPlexSerif-Regular.ttf",
+        "ofl/inter/Inter[slnt,wght].ttf",
+        "ofl/istokweb/IstokWeb-Italic.ttf",
+        "ofl/istokweb/IstokWeb-Regular.ttf",
+        "ofl/jejugothic/JejuGothic-Regular.ttf",
+        "ofl/jejuhallasan/JejuHallasan-Regular.ttf",
+        "ofl/jejumyeongjo/JejuMyeongjo-Regular.ttf",
+        "ofl/jetbrainsmono/JetBrainsMono-Italic[wght].ttf",
+        "ofl/jetbrainsmono/JetBrainsMono[wght].ttf",
+        "ofl/jost/Jost-Italic[wght].ttf",
+        "ofl/jost/Jost[wght].ttf",
+        "ofl/kaiseidecol/KaiseiDecol-Regular.ttf",
+        "ofl/kaiseiharunoumi/KaiseiHarunoUmi-Regular.ttf",
+        "ofl/kaiseiopti/KaiseiOpti-Regular.ttf",
+        "ofl/kaiseitokumin/KaiseiTokumin-Regular.ttf",
+        "ofl/kellyslab/KellySlab-Regular.ttf",
+        "ofl/kiwimaru/KiwiMaru-Regular.ttf",
+        "ofl/kleeone/KleeOne-Regular.ttf",
+        "ofl/kopubbatang/KoPubBatang-Regular.ttf",
+        "ofl/kurale/Kurale-Regular.ttf",
+        "ofl/lato/Lato-Italic.ttf",
+        "ofl/lato/Lato-Regular.ttf",
+        "ofl/ledger/Ledger-Regular.ttf",
+        "ofl/literata/Literata-Italic[opsz,wght].ttf",
+        "ofl/literata/Literata[opsz,wght].ttf",
+        "ofl/lobster/Lobster-Regular.ttf",
+        "ofl/lora/Lora-Italic[wght].ttf",
+        "ofl/lora/Lora[wght].ttf",
+        "ofl/marckscript/MarckScript-Regular.ttf",
+        "ofl/marmelad/Marmelad-Regular.ttf",
+        "ofl/merriweather/Merriweather-Italic.ttf",
+        "ofl/merriweather/Merriweather-Regular.ttf",
+        "ofl/montserratalternates/MontserratAlternates-Italic.ttf",
+        "ofl/montserratalternates/MontserratAlternates-Regular.ttf",
+        "ofl/montserrat/Montserrat-Italic.ttf",
+        "ofl/montserrat/Montserrat-Regular.ttf",
+        "ofl/mplus1p/Mplus1p-Regular.ttf",
+        "ofl/mulish/Mulish-Italic[wght].ttf",
+        "ofl/nanumgothiccoding/NanumGothicCoding-Regular.ttf",
+        "ofl/neucha/Neucha.ttf",
+        "ofl/newscycle/NewsCycle-Regular.ttf",
+        "ofl/nobile/Nobile-Italic.ttf",
+        "ofl/nobile/Nobile-Regular.ttf",
+        "ofl/nunito/Nunito-Italic[wght].ttf",
+        "ofl/nunitosans/NunitoSans-Italic.ttf",
+        "ofl/nunitosans/NunitoSans-Regular.ttf",
+        "ofl/oi/Oi-Regular.ttf",
+        "ofl/oranienbaum/Oranienbaum-Regular.ttf",
+        "ofl/orelegaone/OrelegaOne-Regular.ttf",
+        "ofl/oswald/Oswald[wght].ttf",
+        "ofl/overpass/Overpass-Italic[wght].ttf",
+        "ofl/overpass/Overpass[wght].ttf",
+        "ofl/pacifico/Pacifico-Regular.ttf",
+        "ofl/pangolin/Pangolin-Regular.ttf",
+        "ofl/pattaya/Pattaya-Regular.ttf",
+        "ofl/philosopher/Philosopher-Italic.ttf",
+        "ofl/philosopher/Philosopher-Regular.ttf",
+        "ofl/piazzolla/Piazzolla-Italic[opsz,wght].ttf",
+        "ofl/playfairdisplay/PlayfairDisplay-Italic[wght].ttf",
+        "ofl/playfairdisplay/PlayfairDisplay[wght].ttf",
+        "ofl/playfairdisplaysc/PlayfairDisplaySC-Italic.ttf",
+        "ofl/playfairdisplaysc/PlayfairDisplaySC-Regular.ttf",
+        "ofl/play/Play-Regular.ttf",
+        "ofl/podkova/Podkova[wght].ttf",
+        "ofl/podkovavfbeta/PodkovaVFBeta.ttf",
+        "ofl/poiretone/PoiretOne-Regular.ttf",
+        "ofl/prata/Prata-Regular.ttf",
+        "ofl/pressstart2p/PressStart2P-Regular.ttf",
+        "ofl/prostoone/ProstoOne-Regular.ttf",
+        "ofl/pushster/Pushster-Regular.ttf",
+        "ofl/raleway/Raleway-Italic[wght].ttf",
+        "ofl/rampartone/RampartOne-Regular.ttf",
+        "ofl/reggaeone/ReggaeOne-Regular.ttf",
+        "ofl/robotoflex/RobotoFlex[GRAD,XOPQ,XTRA,YOPQ,YTAS,YTDE,YTFI,YTLC,YTUC,opsz,slnt,wdth,wght].ttf",
+        "ofl/rocknrollone/RocknRollOne-Regular.ttf",
+        "ofl/roundedmplus1c/RoundedMplus1c-Regular.ttf",
+        "ofl/rubikbeastly/RubikBeastly-Regular.ttf",
+        "ofl/rubikmonoone/RubikMonoOne-Regular.ttf",
+        "ofl/rubikone/RubikOne-Regular.ttf",
+        "ofl/rubik/Rubik-Italic[wght].ttf",
+        "ofl/ruda/Ruda[wght].ttf",
+        "ofl/ruslandisplay/RuslanDisplay.ttf",
+        "ofl/russoone/RussoOne-Regular.ttf",
+        "ofl/sawarabigothic/SawarabiGothic-Regular.ttf",
+        "ofl/scada/Scada-Italic.ttf",
+        "ofl/scada/Scada-Regular.ttf",
+        "ofl/seoulhangangcondensed/SeoulHangangCondensed-BoldL.ttf",
+        "ofl/seoulhangangcondensed/SeoulHangangCondensed-Bold.ttf",
+        "ofl/seoulhangangcondensed/SeoulHangangCondensed-ExtraBold.ttf",
+        "ofl/seoulhangangcondensed/SeoulHangangCondensed-Medium.ttf",
+        "ofl/seoulhangang/SeoulHangang-Bold.ttf",
+        "ofl/seoulhangang/SeoulHangang-ExtraBold.ttf",
+        "ofl/seoulhangang/SeoulHangang-Medium.ttf",
+        "ofl/seoulnamsancondensed/SeoulNamsanCondensed-Black.ttf",
+        "ofl/seoulnamsancondensed/SeoulNamsanCondensed-Bold.ttf",
+        "ofl/seoulnamsancondensed/SeoulNamsanCondensed-ExtraBold.ttf",
+        "ofl/seoulnamsancondensed/SeoulNamsanCondensed-Medium.ttf",
+        "ofl/seoulnamsan/SeoulNamsan-Bold.ttf",
+        "ofl/seoulnamsan/SeoulNamsan-ExtraBold.ttf",
+        "ofl/seoulnamsan/SeoulNamsan-Medium.ttf",
+        "ofl/seoulnamsanvertical/SeoulNamsanVertical-Regular.ttf",
+        "ofl/seymourone/SeymourOne-Regular.ttf",
+        "ofl/sofiasans/SofiaSans-Italic[wdth,wght].ttf",
+        "ofl/sofiasans/SofiaSans[wdth,wght].ttf",
+        "ofl/sourcesans3/SourceSans3-Italic[wght].ttf",
+        "ofl/sourcesans3/SourceSans3[wght].ttf",
+        "ofl/sourcesanspro/SourceSansPro-Regular.ttf",
+        "ofl/sourceserifpro/SourceSerifPro-Italic.ttf",
+        "ofl/sourceserifpro/SourceSerifPro-Regular.ttf",
+        "ofl/spectralsc/SpectralSC-Italic.ttf",
+        "ofl/spectralsc/SpectralSC-Regular.ttf",
+        "ofl/spectral/Spectral-Italic.ttf",
+        "ofl/spectral/Spectral-Regular.ttf",
+        "ofl/stalinistone/StalinistOne-Regular.ttf",
+        "ofl/stick/Stick-Regular.ttf",
+        "ofl/stixtwomath/STIXTwoMath-Regular.ttf",
+        "ofl/stixtwotext/STIXTwoText-Italic[wght].ttf",
+        "ofl/stixtwotext/STIXTwoText[wght].ttf",
+        "ofl/strong/Strong-Regular.ttf",
+        "ofl/tenorsans/TenorSans-Regular.ttf",
+        "ofl/trainone/TrainOne-Regular.ttf",
+        "ofl/tuffy/Tuffy-Italic.ttf",
+        "ofl/tuffy/Tuffy-Regular.ttf",
+        "ofl/underdog/Underdog-Regular.ttf",
+        "ofl/viaodalibre/ViaodaLibre-Regular.ttf",
+        "ofl/vollkornsc/VollkornSC-Regular.ttf",
+        "ofl/vollkorn/Vollkorn-Italic[wght].ttf",
+        "ofl/vollkorn/Vollkorn[wght].ttf",
+        "ofl/yesevaone/YesevaOne-Regular.ttf",
+        "ofl/yomogi/Yomogi-Regular.ttf",
+        "ofl/yujiboku/YujiBoku-Regular.ttf",
+        "ofl/yujimai/YujiMai-Regular.ttf",
+        "ofl/yujisyuku/YujiSyuku-Regular.ttf",
+        "ofl/zenantiquesoft/ZenAntiqueSoft-Regular.ttf",
+        "ofl/zenantique/ZenAntique-Regular.ttf",
+        "ofl/zenkakugothicantique/ZenKakuGothicAntique-Regular.ttf",
+        "ofl/zenkakugothicnew/ZenKakuGothicNew-Regular.ttf",
+        "ofl/zenkurenaido/ZenKurenaido-Regular.ttf",
+        "ofl/zenmarugothic/ZenMaruGothic-Regular.ttf",
+        "ofl/zenoldmincho/ZenOldMincho-Regular.ttf"
+    ],
+    "tamil": [
+        "ofl/arimamadurai/ArimaMadurai-Regular.ttf",
+        "ofl/baloothambi2/BalooThambi2-Regular.ttf",
+        "ofl/coiny/Coiny-Regular.ttf",
+        "ofl/hindmadurai/HindMadurai-Regular.ttf",
+        "ofl/kavivanar/Kavivanar-Regular.ttf",
+        "ofl/meerainimai/MeeraInimai-Regular.ttf",
+        "ofl/muktamalar/MuktaMalar-Regular.ttf",
+        "ofl/oi/Oi-Regular.ttf",
+        "ofl/pavanam/Pavanam-Regular.ttf",
+        "ofl/postnobillsjaffna/PostNoBillsJaffna-Regular.ttf"
+    ],
+    "telugu": [
+        "ofl/akayatelivigala/AkayaTelivigala-Regular.ttf",
+        "ofl/balootammudu2/BalooTammudu2[wght].ttf",
+        "ofl/chathura/Chathura-Regular.ttf",
+        "ofl/dhurjati/Dhurjati-Regular.ttf",
+        "ofl/gidugu/Gidugu-Regular.ttf",
+        "ofl/gurajada/Gurajada-Regular.ttf",
+        "ofl/hindguntur/HindGuntur-Regular.ttf",
+        "ofl/lakkireddy/LakkiReddy-Regular.ttf",
+        "ofl/mallanna/Mallanna-Regular.ttf",
+        "ofl/mandali/Mandali-Regular.ttf",
+        "ofl/nats/NATS-Regular.ttf",
+        "ofl/ntr/NTR-Regular.ttf",
+        "ofl/peddana/Peddana-Regular.ttf",
+        "ofl/ramabhadra/Ramabhadra-Regular.ttf",
+        "ofl/ramaraja/Ramaraja-Regular.ttf",
+        "ofl/raviprakash/RaviPrakash-Regular.ttf",
+        "ofl/sreekrushnadevaraya/SreeKrushnadevaraya-Regular.ttf",
+        "ofl/suranna/Suranna-Regular.ttf",
+        "ofl/suravaram/Suravaram-Regular.ttf",
+        "ofl/tenaliramakrishna/TenaliRamakrishna-Regular.ttf",
+        "ofl/timmana/Timmana-Regular.ttf"
+    ],
+    "thai": [
+        "ofl/athiti/Athiti-Regular.ttf",
+        "ofl/baijamjuree/BaiJamjuree-Italic.ttf",
+        "ofl/baijamjuree/BaiJamjuree-Regular.ttf",
+        "ofl/chakrapetch/ChakraPetch-Italic.ttf",
+        "ofl/chakrapetch/ChakraPetch-Regular.ttf",
+        "ofl/charm/Charm-Regular.ttf",
+        "ofl/charmonman/Charmonman-Regular.ttf",
+        "ofl/chonburi/Chonburi-Regular.ttf",
+        "ofl/fahkwang/Fahkwang-Italic.ttf",
+        "ofl/fahkwang/Fahkwang-Regular.ttf",
+        "ofl/ibmplexsansthai/IBMPlexSansThai-Regular.ttf",
+        "ofl/ibmplexsansthailooped/IBMPlexSansThaiLooped-Regular.ttf",
+        "ofl/itim/Itim-Regular.ttf",
+        "ofl/k2d/K2D-Italic.ttf",
+        "ofl/k2d/K2D-Regular.ttf",
+        "ofl/kanit/Kanit-Italic.ttf",
+        "ofl/kanit/Kanit-Regular.ttf",
+        "ofl/kodchasan/Kodchasan-Italic.ttf",
+        "ofl/kodchasan/Kodchasan-Regular.ttf",
+        "ofl/koho/KoHo-Italic.ttf",
+        "ofl/koho/KoHo-Regular.ttf",
+        "ofl/krub/Krub-Italic.ttf",
+        "ofl/krub/Krub-Regular.ttf",
+        "ofl/maitree/Maitree-Regular.ttf",
+        "ofl/mali/Mali-Italic.ttf",
+        "ofl/mali/Mali-Regular.ttf",
+        "ofl/mitr/Mitr-Regular.ttf",
+        "ofl/niramit/Niramit-Italic.ttf",
+        "ofl/niramit/Niramit-Regular.ttf",
+        "ofl/pattaya/Pattaya-Regular.ttf",
+        "ofl/pridi/Pridi-Regular.ttf",
+        "ofl/prompt/Prompt-Italic.ttf",
+        "ofl/prompt/Prompt-Regular.ttf",
+        "ofl/sarabun/Sarabun-Italic.ttf",
+        "ofl/sarabun/Sarabun-Regular.ttf",
+        "ofl/sriracha/Sriracha-Regular.ttf",
+        "ofl/srisakdi/Srisakdi-Regular.ttf",
+        "ofl/taviraj/Taviraj-Italic.ttf",
+        "ofl/taviraj/Taviraj-Regular.ttf",
+        "ofl/thasadith/Thasadith-Italic.ttf",
+        "ofl/thasadith/Thasadith-Regular.ttf",
+        "ofl/trirong/Trirong-Italic.ttf",
+        "ofl/trirong/Trirong-Regular.ttf"
+    ],
+    "chinese": [
+        "ofl/liujianmaocao/LiuJianMaoCao-Regular.ttf",
+        "ofl/longcang/LongCang-Regular.ttf",
+        "ofl/mashanzheng/MaShanZheng-Regular.ttf",
+        "ofl/mochiypopone/MochiyPopOne-Regular.ttf",
+        "ofl/mochiypoppone/MochiyPopPOne-Regular.ttf",
+        "ofl/mplus1code/MPLUS1Code[wght].ttf",
+        "ofl/mplus1p/Mplus1p-Regular.ttf",
+        "ofl/newtegomin/NewTegomin-Regular.ttf",
+        "ofl/pottaone/PottaOne-Regular.ttf",
+        "ofl/rampartone/RampartOne-Regular.ttf",
+        "ofl/reggaeone/ReggaeOne-Regular.ttf"
+    ]
+}

font_list_noto_sans.json

@@ -0,0 +1,41 @@
+{
+    "arabic": [
+        "notosans/notosansarabic/NotoSansArabic[wdth,wght].ttf"
+    ],
+    "bengali": [
+        "notosans/notosansbengali/NotoSansBengali[wdth,wght].ttf"
+    ],
+    "gujarati": [
+        "notosans/notosansgujarati/NotoSansGujarati-Regular.ttf"
+    ],
+    "hebrew": [
+        "notosans/notosanshebrew/NotoSansHebrew[wdth,wght].ttf"
+    ],
+    "japanese": [
+        "notosans/notosansjp/NotoSansJP-Regular.otf"
+    ],
+    "khmer": [
+        "notosans/notosanskhmer/NotoSansKhmer[wdth,wght].ttf"
+    ],
+    "korean": [
+        "notosans/notosanskr/NotoSansKR-Regular.otf"
+    ],
+    "malayalam": [
+        "notosans/notosansmalayalam/NotoSansMalayalam[wdth,wght].ttf"
+    ],
+    "cyrillic": [
+        "notosans/notosans/NotoSans-Regular.ttf"
+    ],
+    "tamil": [
+        "notosans/notosanstamil/NotoSansTamil[wdth,wght].ttf"
+    ],
+    "telugu": [
+        "notosans/notosanstelugu/NotoSansTelugu[wdth,wght].ttf"
+    ],
+    "thai": [
+        "notosans/notosansthai/NotoSansThai[wdth,wght].ttf"
+    ],
+    "chinese": [
+        "notosans/notosanssc/NotoSansSC-Regular.otf"
+    ]
+}

inference.py

@@ -0,0 +1,163 @@
+from pathlib import Path
+from typing import Dict, List, Union, Tuple
+
+from omegaconf import OmegaConf
+import numpy as np
+import torch
+from torch import nn
+from PIL import Image, ImageDraw, ImageFont
+
+import models
+
+GENERATOR_PREFIX = "networks.g."
+WHITE = 255
+EXAMPLE_CHARACTERS = ['A', 'B', 'C', 'D', 'E']
+
+class InferenceServicer:
+    def __init__(self, hp, checkpoint_path, content_image_dir, imsize=64, gpu_id='0') -> None:
+        self.hp = hp
+        self.imsize = imsize
+        
+        if gpu_id is None:
+            self.device = torch.device(f'cuda:0') if torch.cuda.is_available() else 'cpu'
+        else:
+            self.device = torch.device(f'cuda:{gpu_id}')
+        
+        model_config = self.hp.models.G
+        self.model: nn.Module = models.Generator(model_config)
+        
+        # Load Generator model weight
+        model_state_dict_pl = torch.load(checkpoint_path, map_location='cpu')
+        generator_state_dict = self.convert_generator_state_dict(model_state_dict_pl)
+        self.model.load_state_dict(generator_state_dict)
+        self.model.to(device=self.device)
+        self.model.eval()
+        
+        # Setting Content font files
+        self.content_character_dict = self.load_content_character_dict(Path(content_image_dir))
+    
+    @staticmethod
+    def convert_generator_state_dict(model_state_dict_pl):
+        generator_prefix = GENERATOR_PREFIX
+        generator_state_dict = {}
+        for module_name, module_state in model_state_dict_pl['state_dict'].items():
+            if module_name.startswith(generator_prefix):
+                generator_state_dict[module_name[len(generator_prefix):]] = module_state
+    
+        return generator_state_dict
+    
+    @staticmethod
+    def load_content_character_dict(content_image_dir: Path) -> Dict[str, Path]:
+        content_character_dict = {}
+        for filepath in content_image_dir.glob("**/*.png"):
+            content_character_dict[filepath.stem] = filepath
+        return content_character_dict
+    
+    @staticmethod
+    def center_align(bg_img: Image.Image, item_img: Image.Image, fit=False) -> Image.Image:
+        bg_img = bg_img.copy()
+        item_img = item_img.copy()
+        item_w, item_h = item_img.size
+        W, H = bg_img.size
+        if fit:
+            item_ratio = item_w / item_h
+            bg_ratio = W / H
+
+            if bg_ratio > item_ratio:
+                # height fitting
+                resize_ratio = H / item_h
+            else:
+                # width fitting
+                resize_ratio = W / item_w
+            item_img = item_img.resize((int(item_w * resize_ratio), int(item_h * resize_ratio)))
+            item_w, item_h = item_img.size
+
+        bg_img.paste(item_img, ((W - item_w) // 2, (H - item_h) // 2))
+        return bg_img
+        
+    def set_image(self, image: Union[Path, Image.Image]) -> Image.Image:
+        if isinstance(image, (str, Path)):
+            image = Image.open(image)
+        assert isinstance(image, Image.Image)
+        
+        bg_img = Image.new('RGB', (self.imsize, self.imsize), color='white')
+        blend_img = self.center_align(bg_img, image, fit=True)
+        return blend_img
+    
+    @staticmethod
+    def pil_image_to_array(blend_img: Image.Image) -> np.ndarray:
+        normalized_array = np.mean(np.array(blend_img, dtype=np.float32), axis=-1) / WHITE  # L-only image normalized to [0, 1]
+        return normalized_array
+    
+    def get_images_from_fontfile(self, font_file_path: Path, imgmode: str = 'RGB', position: tuple = (0, 0), font_size: int = 128, padding: int = 100) -> List[Image.Image]:
+        
+        imagefont = ImageFont.truetype(str(font_file_path), size=font_size)
+        example_characters = EXAMPLE_CHARACTERS
+
+        font_images: List[Image.Image] = []
+        
+        for character in example_characters:
+            x, y, _, _ = imagefont.getbbox(character)
+            img = Image.new(imgmode, (x + padding, y + padding), color='white')
+            draw = ImageDraw.Draw(img)
+            
+            # bbox = draw.textbbox((0,0), character, font=imagefont)
+            # w = bbox[2] - bbox[0]
+            # h = bbox[3] - bbox[1]
+            
+            w, h = draw.textsize(character, font=imagefont)
+            
+            img = Image.new(imgmode, (w + padding, h + padding), color='white')
+            draw = ImageDraw.Draw(img)
+            draw.text(position, text=character, font=imagefont, fill='black')
+            img = img.convert(imgmode)
+            font_images.append(img)
+
+        return font_images
+    
+    @staticmethod
+    def get_hex_from_char(char: str) -> str:
+        assert len(char) == 1
+        return f"{ord(char):04X}".upper()  # 4-digit hex string
+    
+    @torch.no_grad()
+    def inference(self, content_char: str, style_font: Union[str, Path]) -> Tuple[Image.Image, List[Image.Image], Image.Image]:
+        assert len(content_char) > 0
+        content_char = content_char[:1]  # only get the first character if the length > 1
+        char_hex = self.get_hex_from_char(content_char)
+        
+        if char_hex not in self.content_character_dict:
+            raise ValueError(f"The character {content_char} (hex: {char_hex}) is not supported in this model!")
+        
+        content_image = self.set_image(self.content_character_dict[char_hex])
+        style_images: List[Image.Image] = self.get_images_from_fontfile(Path(style_font))
+        style_images: List[Image.Image] = [self.set_image(image) for image in style_images]
+        
+        content_image_array = self.pil_image_to_array(content_image)[np.newaxis, np.newaxis, ...]  # 1 x C(=1) x H x W
+        style_images_array: np.ndarray = np.array([self.pil_image_to_array(image) for image in style_images])[np.newaxis, ...]  # 1 x C(=5, # shots) x H x W, k-shots goes to batch
+        
+        content_input_tensor = torch.from_numpy(content_image_array).to(self.device)
+        style_input_tensor = torch.from_numpy(style_images_array).to(self.device)
+        
+        generated_images: torch.Tensor = self.model((content_input_tensor, style_input_tensor))
+        generated_images = torch.clip(generated_images, 0, 1)
+        assert generated_images.size(0) == 1
+        
+        generated_image_numpy = (generated_images[0].cpu().numpy() * 255).astype(np.uint8)[0, ...]  # H x W
+        return content_image, style_images, Image.fromarray(generated_image_numpy, mode='L')
+
+
+if __name__ == '__main__':
+    hp = OmegaConf.load("config/models/google-font.yaml")
+    checkpoint_path = "epoch=199-step=257400.ckpt"
+    content_image_dir = "../DATA/NotoSans"
+    
+    servicer = InferenceServicer(hp, checkpoint_path, content_image_dir)
+    
+    style_font = "example_fonts/MaShanZheng-Regular.ttf"
+    content_image, style_images, result = servicer.inference("7", style_font)
+    
+    content_image.save("result_content.png")
+    for idx, style_image in enumerate(style_images):
+        style_image.save(f"result_style_{idx:02d}.png")
+    result.save("result_generated.png")

lightning.py

@@ -0,0 +1,313 @@
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import pytorch_lightning as pl
+import importlib
+import PIL.Image as Image
+
+import models
+import datasets
+from evaluator.ssim import SSIM, MSSSIM
+import lpips
+from models.loss import GANHingeLoss
+from utils import set_logger, magic_image_handler
+
+NUM_TEST_SAVE_IMAGE = 10
+
+
+class FontLightningModule(pl.LightningModule):
+    def __init__(self, args):
+        super().__init__()
+        self.args = args
+
+        self.losses = {}
+        self.metrics = {}
+        self.networks = nn.ModuleDict(self.build_models())
+        self.module_keys = list(self.networks.keys())
+
+        self.losses = self.build_losses()
+        self.metrics = self.build_metrics()
+
+        self.opt_tag = {key: None for key in self.networks.keys()}
+        self.sched_tag = {key: None for key in self.networks.keys()}
+        self.sched_use = False
+        # self.automatic_optimization = False
+
+        self.train_d_content = True
+        self.train_d_style = True
+
+    def build_models(self):
+        networks = {}
+        for key, hp_model in self.args.models.items():
+            key_ = key.lower()
+            if 'g' == key_[0]:
+                model_ = models.Generator(hp_model)
+            elif 'd' == key_[0]:
+                model_ = models.PatchGANDiscriminator(hp_model)  # TODO: add option for selecting discriminator
+            else:
+                raise ValueError(f"No key such as {key}")
+
+            networks[key.lower()] = model_
+        return networks
+
+    def build_losses(self):
+        losses_dict = {}
+        losses_dict['L1'] = torch.nn.L1Loss()
+
+        if 'd_content' in self.module_keys:
+            losses_dict['GANLoss_content'] = GANHingeLoss()
+        if 'd_style' in self.module_keys:
+            losses_dict['GANLoss_style'] = GANHingeLoss()
+
+        return losses_dict
+
+    def build_metrics(self):
+        metrics_dict = nn.ModuleDict()
+        metrics_dict['ssim'] = SSIM(val_range=1)  # img value is in [0, 1]
+        metrics_dict['msssim'] = MSSSIM(weights=[0.45, 0.3, 0.25], val_range=1)  # since imsize=64, len(weight)<=3
+        metrics_dict['lpips'] = lpips.LPIPS(net='vgg')
+        return metrics_dict
+
+    def configure_optimizers(self):
+        optims = {}
+        for key, args_model in self.args.models.items():
+            key = key.lower()
+            if args_model['optim'] is not None:
+                args_optim = args_model['optim']
+                module, cls = args_optim['class'].rsplit(".", 1)
+                O = getattr(importlib.import_module(module, package=None), cls)
+                o = O([p for p in self.networks[key].parameters() if p.requires_grad],
+                      lr=args_optim.lr, betas=args_optim.betas)
+
+                optims[key] = o
+
+        optim_module_keys = optims.keys()
+
+        count = 0
+        optim_list = []
+
+        for _key in self.module_keys:
+            if _key in optim_module_keys:
+                optim_list.append(optims[_key])
+                self.opt_tag[_key] = count
+                count += 1
+
+        return optim_list
+
+    def forward(self, content_images, style_images):
+        return self.networks['g']((content_images, style_images))
+
+    def common_forward(self, batch, batch_idx):
+        loss = {}
+        logs = {}
+
+        content_images = batch['content_images']
+        style_images = batch['style_images']
+        gt_images = batch['gt_images']
+        image_paths = batch['image_paths']
+        char_idx = batch['char_idx']
+
+        generated_images = self(content_images, style_images)
+
+        # l1 loss
+        loss['g_L1'] = self.losses['L1'](generated_images, gt_images)
+        loss['g_backward'] = loss['g_L1'] * self.args.logging.lambda_L1
+
+        # loss for training generator
+        if 'd_content' in self.module_keys:
+            loss = self.d_content_loss_for_G(content_images, generated_images, loss)
+
+        if 'd_style' in self.networks.keys():
+            loss = self.d_style_loss_for_G(style_images, generated_images, loss)
+
+        # loss for training discriminator
+        generated_images = generated_images.detach()
+
+        if 'd_content' in self.module_keys:
+            if self.train_d_content:
+                loss = self.d_content_loss_for_D(content_images, generated_images, gt_images, loss)
+
+        if 'd_style' in self.module_keys:
+            if self.train_d_style:
+                loss = self.d_style_loss_for_D(style_images, generated_images, gt_images, loss)
+
+        logs['content_images'] = content_images
+        logs['style_images'] = style_images
+        logs['gt_images'] = gt_images
+        logs['generated_images'] = generated_images
+
+        return loss, logs
+
+    @property
+    def automatic_optimization(self):
+        return False
+
+    def training_step(self, batch, batch_idx):
+        metrics = {}
+        # forward
+        loss, logs = self.common_forward(batch, batch_idx)
+
+        if self.global_step % self.args.logging.freq['train'] == 0:
+            with torch.no_grad():
+                metrics.update(self.calc_metrics(logs['gt_images'], logs['generated_images']))
+
+        # backward
+        opts = self.optimizers()
+
+        opts[self.opt_tag['g']].zero_grad()
+        self.manual_backward(loss['g_backward'])
+
+        if 'd_content' in self.module_keys:
+            if self.train_d_content:
+                opts[self.opt_tag['d_content']].zero_grad()
+                self.manual_backward(loss['dcontent_backward'])
+
+        if 'd_style' in self.module_keys:
+            if self.train_d_style:
+                opts[self.opt_tag['d_style']].zero_grad()
+                self.manual_backward(loss['dstyle_backward'])
+
+        opts[self.opt_tag['g']].step()
+
+        if 'd_content' in self.module_keys:
+            if self.train_d_content:
+                opts[self.opt_tag['d_content']].step()
+
+        if 'd_style' in self.module_keys:
+            if self.train_d_style:
+                opts[self.opt_tag['d_style']].step()
+
+        if self.global_step % self.args.logging.freq['train'] == 0:
+            self.custom_log(loss, metrics, logs, mode='train')
+
+    def validation_step(self, batch, batch_idx):
+        metrics = {}
+        loss, logs = self.common_forward(batch, batch_idx)
+        self.custom_log(loss, metrics, logs, mode='eval')
+
+    def test_step(self, batch, batch_idx):
+        metrics = {}
+        loss, logs = self.common_forward(batch, batch_idx)
+        metrics.update(self.calc_metrics(logs['gt_images'], logs['generated_images']))
+
+        if batch_idx < NUM_TEST_SAVE_IMAGE:
+            for key, value in logs.items():
+                if 'image' in key:
+                    sample_images = (magic_image_handler(value) * 255)[..., 0].astype(np.uint8)
+                    Image.fromarray(sample_images).save(f"{batch_idx:02d}_{key}.png")
+
+        return loss, logs, metrics
+
+    def test_epoch_end(self, test_step_outputs):
+        # do something with the outputs of all test batches
+        # all_test_preds = test_step_outputs.metrics
+        ssim_list = []
+        msssim_list = []
+
+        for _, test_output in enumerate(test_step_outputs):
+
+            ssim_list.append(test_output[2]['SSIM'].cpu().numpy())
+            msssim_list.append(test_output[2]['MSSSIM'].cpu().numpy())
+
+        print(f"SSIM: {np.mean(ssim_list)}")
+        print(f"MSSSIM: {np.mean(msssim_list)}")
+
+    def common_dataloader(self, mode='train', batch_size=None):
+        dataset_cls = getattr(datasets, self.args.datasets.type)
+        dataset_config = getattr(self.args.datasets, mode)
+        dataset = dataset_cls(dataset_config, mode=mode)
+        _batch_size = batch_size if batch_size is not None else dataset_config.batch_size
+        dataloader = DataLoader(dataset,
+                                shuffle=dataset_config.shuffle,
+                                batch_size=_batch_size,
+                                num_workers=dataset_config.num_workers,
+                                drop_last=True)
+
+        return dataloader
+
+    def train_dataloader(self):
+        return self.common_dataloader(mode='train')
+
+    def val_dataloader(self):
+        return self.common_dataloader(mode='eval')
+
+    def test_dataloader(self):
+        return self.common_dataloader(mode='eval')
+
+    def calc_metrics(self, gt_images, generated_images):
+        """
+
+        :param gt_images:
+        :param generated_images:
+        :return:
+        """
+        metrics = {}
+        _gt = torch.clamp(gt_images.clone(), 0, 1)
+        _gen = torch.clamp(generated_images.clone(), 0, 1)
+        metrics['SSIM'] = self.metrics['ssim'](_gt, _gen)
+        msssim_value = self.metrics['msssim'](_gt, _gen)
+        metrics['MSSSIM'] = msssim_value if not torch.isnan(msssim_value) else torch.tensor(0.).type_as(_gt)
+        metrics['LPIPS'] = self.metrics['lpips'](_gt * 2 - 1, _gen * 2 - 1).squeeze().mean()
+        return metrics
+
+    # region step
+    def d_content_loss_for_G(self, content_images, generated_images, loss):
+        pred_generated = self.networks['d_content'](torch.cat([content_images, generated_images], dim=1))
+        loss['g_gan_content'] = self.losses['GANLoss_content'](pred_generated, True, for_discriminator=False)
+
+        loss['g_backward'] += loss['g_gan_content']
+        return loss
+
+    def d_content_loss_for_D(self, content_images, generated_images, gt_images, loss):
+        # D
+        if 'd_content' in self.module_keys:
+            if self.train_d_content:
+                pred_gt_images = self.networks['d_content'](torch.cat([content_images, gt_images], dim=1))
+                pred_generated_images = self.networks['d_content'](torch.cat([content_images, generated_images], dim=1))
+
+                loss['dcontent_gt'] = self.losses['GANLoss_content'](pred_gt_images, True, for_discriminator=True)
+                loss['dcontent_gen'] = self.losses['GANLoss_content'](pred_generated_images, False, for_discriminator=True)
+                loss['dcontent_backward'] = (loss['dcontent_gt'] + loss['dcontent_gen'])
+
+        return loss
+
+    def d_style_loss_for_G(self, style_images, generated_images, loss):
+        pred_generated = self.networks['d_style'](torch.cat([style_images, generated_images], dim=1))
+        loss['g_gan_style'] = self.losses['GANLoss_style'](pred_generated, True, for_discriminator=False)
+
+        assert self.train_d_style
+        loss['g_backward'] += loss['g_gan_style']
+        return loss
+
+    def d_style_loss_for_D(self, style_images, generated_images, gt_images, loss):
+        pred_gt_images = self.networks['d_style'](torch.cat([style_images, gt_images], dim=1))
+        pred_generated_images = self.networks['d_style'](torch.cat([style_images, generated_images], dim=1))
+
+        loss['dstyle_gt'] = self.losses['GANLoss_style'](pred_gt_images, True, for_discriminator=True)
+        loss['dstyle_gen'] = self.losses['GANLoss_style'](pred_generated_images, False, for_discriminator=True)
+        loss['dstyle_backward'] = (loss['dstyle_gt'] + loss['dstyle_gen'])
+
+        return loss
+
+    def custom_log(self, loss, metrics, logs, mode):
+        # logging values with tensorboard
+        for loss_full_key, value in loss.items():
+            model_type, loss_type = loss_full_key.split('_')[0], "_".join(loss_full_key.split('_')[1:])
+            self.log(f'{model_type}/{mode}_{loss_type}', value)
+
+        for metric_full_key, value in metrics.items():
+            model_type, metric_type = metric_full_key.split('_')[0], "_".join(metric_full_key.split('_')[1:])
+            self.log(f'{model_type}/{mode}_{metric_type}', value)
+
+        # logging images, params, etc.
+        tensorboard = self.logger.experiment
+        for key, value in logs.items():
+            if 'image' in key:
+                sample_images = magic_image_handler(value)
+                tensorboard.add_image(f"{mode}/" + key, sample_images, self.global_step, dataformats='HWC')
+            elif 'param' in key:
+                tensorboard.add_histogram(f"{mode}" + key, value, self.global_step)
+            else:
+                raise RuntimeError(f"Only logging with one of keywords: image, param | current input: {key}")

models/__init__.py

@@ -0,0 +1,2 @@
+from .generator import *
+from .discriminator import *

models/decoder.py

@@ -0,0 +1,45 @@
+import torch
+import torch.nn as nn
+from models.module import ResidualBlocks
+
+_DECODER_CHANNEL_DEFAULT = 512
+
+
+class Decoder(nn.Module):
+    def __init__(self, hp, in_channels=_DECODER_CHANNEL_DEFAULT, out_channels=1):
+        super().__init__()
+        self.module = nn.ModuleList()
+
+    def forward(self, x):
+        for block in self.module:
+            x = block(x)
+        return x
+
+
+class VanillaDecoder(Decoder):
+    def __init__(self, hp, in_channels, out_channels):
+        super().__init__(hp, in_channels, out_channels)
+        self.depth = hp.decoder.depth
+        self.blocks = hp.decoder.residual_blocks
+
+        self.module = nn.ModuleList()
+        if self.blocks > 0:
+            self.module.append(ResidualBlocks(in_channels, n_blocks=self.blocks))
+
+        for layer_idx in range(1, self.depth + 1):  # add upsampling layers
+            self.module.append(nn.Sequential(
+                nn.ConvTranspose2d(in_channels // (2 ** (layer_idx - 1)),
+                                   in_channels // (2 ** layer_idx),
+                                   kernel_size=3, stride=2,
+                                   padding=1, output_padding=1,
+                                   bias=False),
+                nn.BatchNorm2d(in_channels // (2 ** layer_idx)),
+                nn.ReLU(True)
+            ))
+
+        final = nn.Sequential(
+            nn.Conv2d(in_channels // (2 ** self.depth), out_channels, kernel_size=7, padding=3, padding_mode='reflect'),
+            nn.Tanh()
+        )
+
+        self.module.append(final)

models/discriminator.py

@@ -0,0 +1,54 @@
+import functools
+
+import omegaconf
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+# FIXME
+
+
+class PatchGANDiscriminator(nn.Module):
+    """Defines a PatchGAN discriminator"""
+
+    def __init__(self, hp, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d):
+        """Construct a PatchGAN discriminator
+
+        Parameters:
+            ndf (int)       -- the number of filters in the last conv layer
+            n_layers (int)  -- the number of conv layers in the discriminator
+            norm_layer      -- normalization layer
+        """
+        super().__init__()
+        self.hp = hp
+        in_channels = hp.in_channels
+
+        if type(norm_layer) == functools.partial:  # no need to use bias as BatchNorm2d has affine parameters
+            use_bias = norm_layer.func == nn.InstanceNorm2d
+        else:
+            use_bias = norm_layer == nn.InstanceNorm2d
+        kw = 4
+        padw = 1
+        sequence = [nn.Conv2d(in_channels, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+        nf_mult = 1
+        nf_mult_prev = 1
+        for n in range(1, n_layers):  # gradually increase the number of filters
+            nf_mult_prev = nf_mult
+            nf_mult = min(2 ** n, 8)
+            sequence += [
+                nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),
+                norm_layer(ndf * nf_mult),
+                nn.LeakyReLU(0.2, True)
+            ]
+        nf_mult_prev = nf_mult
+        nf_mult = min(2 ** n_layers, 8)
+        sequence += [
+            nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),
+            norm_layer(ndf * nf_mult),
+            nn.LeakyReLU(0.2, True)
+        ]
+        sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)]
+        self.model = nn.Sequential(*sequence)
+
+    def forward(self, x):
+        return self.model(x)

models/encoder.py

@@ -0,0 +1,65 @@
+import torch
+import torch.nn as nn
+from models.module import Conv2d, StyleAttentionBlock
+
+_ENCODER_CHANNEL_DEFAULT = 256
+
+
+class Encoder(nn.Module):
+    def __init__(self, hp, in_channels=1, out_channels=_ENCODER_CHANNEL_DEFAULT):
+        super().__init__()
+        self.hp = hp
+        self.module = nn.ModuleList()
+
+    def forward(self, x):
+        for block in self.module:
+            x = block(x)
+        return x
+
+
+class ContentVanillaEncoder(Encoder):
+    def __init__(self, hp, in_channels, out_channels):
+        super().__init__(hp, in_channels, out_channels)
+        self.depth = hp.encoder.content.depth
+        assert out_channels // (2 ** self.depth) >= in_channels * 2, "Output channel should be increased"
+
+        self.module = nn.ModuleList()
+        self.module.append(
+            Conv2d(in_channels, out_channels // (2 ** self.depth),
+                   kernel_size=7, padding=3, padding_mode='reflect', bias=False)
+        )
+
+        for layer_idx in range(1, self.depth + 1):  # downsample
+            self.module.append(
+                Conv2d(out_channels // (2 ** (self.depth - layer_idx + 1)),
+                       out_channels // (2 ** (self.depth - layer_idx)),
+                       kernel_size=3, stride=2, padding=1, bias=False)
+            )
+
+
+class StyleVanillaEncoder(Encoder):
+    def __init__(self, hp, in_channels, out_channels):
+        super().__init__(hp, in_channels, out_channels)
+        self.depth = hp.encoder.style.depth
+        assert out_channels // (2 ** self.depth) >= in_channels * 2, "Output channel should be increased"
+
+        encoder_module = []
+        encoder_module.append(
+            Conv2d(in_channels, out_channels // (2 ** self.depth),
+                   kernel_size=7, padding=3, padding_mode='reflect', bias=False)
+        )
+
+        for layer_idx in range(1, self.depth + 1):  # downsample
+            encoder_module.append(
+                Conv2d(out_channels // (2 ** (self.depth - layer_idx + 1)),
+                       out_channels // (2 ** (self.depth - layer_idx)),
+                       kernel_size=3, stride=2, padding=1, bias=False)
+            )
+        self.add_module("encoder_module", nn.Sequential(*encoder_module))
+        self.add_module("attention_module", StyleAttentionBlock(out_channels))
+
+    def forward(self, x):
+        B, K, H, W = x.size()
+        out = self.encoder_module(x.view(-1, 1, H, W))
+        out = self.attention_module(out, B, K)
+        return out

models/generator.py

@@ -0,0 +1,22 @@
+import torch
+import torch.nn as nn
+from . import encoder, decoder
+
+class Generator(nn.Module):
+    def __init__(self, hp, in_channels=1):
+        super().__init__()
+        self.hp = hp
+        _ngf = 64
+        hidden_dim = _ngf * 4
+        self.content_encoder = getattr(encoder, self.hp.encoder.content.type)(self.hp, in_channels, hidden_dim)
+        self.style_encoder = getattr(encoder, self.hp.encoder.style.type)(self.hp, in_channels, hidden_dim)
+        self.decoder = getattr(decoder, self.hp.decoder.type)(self.hp, hidden_dim * 2, in_channels)
+                
+    def forward(self, images):
+        content_images, style_images = images
+        content_feature = self.content_encoder(content_images)
+        style_images = style_images * 2 - 1  # pixel value range -1 to 1
+        style_feature = self.style_encoder(style_images)  # K-shot as batch
+        _, _, H, W = content_feature.size()
+        out = self.decoder(torch.cat([content_feature, style_feature.expand(-1, -1, H, W)], dim=1))
+        return out

models/loss.py

@@ -0,0 +1,16 @@
+import torch
+import torch.nn as nn
+
+class GANHingeLoss(nn.Module):
+    def __init__(self):
+        super(GANHingeLoss, self).__init__()
+        self.relu = nn.ReLU()
+    
+    def __call__(self, pred, is_real, for_discriminator):
+        if for_discriminator:
+            if is_real:
+                return self.relu(1 - pred).mean()
+            return self.relu(1 + pred).mean()
+        
+        assert is_real, "The generator's hinge loss must be aiming for real"
+        return -1.0 * pred.mean()

models/module.py

@@ -0,0 +1,159 @@
+import torch
+import torch.nn as nn
+
+
+class Conv2d(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1,
+                 padding=0, padding_mode='zeros', bias=True, residual=False):
+        super(Conv2d, self).__init__()
+        self.conv_block = nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, kernel_size, stride,
+                      padding, padding_mode=padding_mode, bias=bias),
+            nn.BatchNorm2d(out_channels)
+        )
+        self.residual = residual
+        self.act = nn.ReLU()
+
+    def forward(self, x):
+        out = self.conv_block(x)
+        if self.residual:
+            out += x
+        out = self.act(out)
+        return out
+
+
+class ResnetBlock(nn.Module):
+    def __init__(self, channel, padding_mode, norm_layer=nn.BatchNorm2d, bias=False):
+        super().__init__()
+        if padding_mode not in ['reflect', 'zero']:
+            raise NotImplementedError(f"{padding_mode} is not supported!")
+
+        self.block = nn.Sequential(
+            nn.Conv2d(channel, channel, kernel_size=3, padding=1, padding_mode=padding_mode, bias=bias),
+            norm_layer(channel)
+        )
+        self.act = nn.ReLU()
+
+    def forward(self, x):
+        out = self.block(x)
+        out = out + x
+        out = self.act(out)
+        return out
+
+
+class ResidualBlocks(nn.Module):
+    def __init__(self, channel, n_blocks=6):
+        super().__init__()
+        model = []
+        for i in range(n_blocks):  # add ResNet blocks
+            model += [ResnetBlock(channel, padding_mode='reflect')]
+
+        self.module = nn.Sequential(*model)
+
+    def forward(self, x):
+        return self.module(x)
+
+
+class SelfAttentionBlock(nn.Module):
+
+    def __init__(self, in_dim):
+        super().__init__()
+        self.feature_dim = in_dim // 8
+        self.query_conv = nn.Conv2d(in_channels=in_dim, out_channels=self.feature_dim, kernel_size=1)
+        self.key_conv = nn.Conv2d(in_channels=in_dim, out_channels=self.feature_dim, kernel_size=1)
+        self.value_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
+        self.gamma = nn.Parameter(torch.zeros(1))
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x):
+        B, C, H, W = x.size()
+        _query = self.query_conv(x).view(B, -1, H * W).permute(0, 2, 1)  # B x C x (H'*W')
+        _key = self.key_conv(x).view(B, -1, H * W)  # B x C x (H'*W')
+        attn_matrix = torch.bmm(_query, _key)
+        attention = self.softmax(attn_matrix)  # B x (H'*W') x (H'*W')
+        _value = self.value_conv(x).view(B, -1, H * W)  # B X C X (H * W)
+
+        out = torch.bmm(_value, attention.permute(0, 2, 1))
+        out = out.view(B, C, H, W)
+
+        out = self.gamma * out + x
+        return out
+
+
+class ContextAwareAttentionBlock(nn.Module):
+
+    def __init__(self, in_channels, hidden_dim=128):
+        super().__init__()
+        self.self_attn = SelfAttentionBlock(in_channels)
+        self.fc = nn.Linear(in_channels, hidden_dim)
+        self.context_vector = nn.Linear(hidden_dim, 1, bias=False)
+        self.softmax = nn.Softmax(dim=1)
+
+    def forward(self, style_features):
+        B, C, H, W = style_features.size()
+        h = self.self_attn(style_features)
+        h = h.permute(0, 2, 3, 1).reshape(-1, C)
+        h = torch.tanh(self.fc(h))  # (B*H*W) x self.hidden_dim
+        h = self.context_vector(h)  # (B*H*W) x 1
+        attention_score = self.softmax(h.view(B, H * W)).view(B, 1, H, W)  # B x 1 x H x W
+        return torch.sum(style_features * attention_score, dim=[2, 3])  # B x C
+
+
+class LayerAttentionBlock(nn.Module):
+    """from FTransGAN
+    """
+
+    def __init__(self, in_channels):
+        super().__init__()
+        self.in_channels = in_channels
+        self.width_feat = 4
+        self.height_feat = 4
+        self.fc = nn.Linear(self.in_channels * self.width_feat * self.height_feat, 3)
+        self.softmax = nn.Softmax(dim=1)
+
+    def forward(self, style_features, style_features_1, style_features_2, style_features_3, B, K):
+        style_features = torch.mean(style_features.view(B, K, self.in_channels, self.height_feat, self.width_feat), dim=1)
+        style_features = style_features.view(B, -1)
+        weight = self.softmax(self.fc(style_features))
+
+        style_features_1 = torch.mean(style_features_1.view(B, K, self.in_channels), dim=1)
+        style_features_2 = torch.mean(style_features_2.view(B, K, self.in_channels), dim=1)
+        style_features_3 = torch.mean(style_features_3.view(B, K, self.in_channels), dim=1)
+
+        style_features = (style_features_1 * weight.narrow(1, 0, 1) +
+                          style_features_2 * weight.narrow(1, 1, 1) +
+                          style_features_3 * weight.narrow(1, 2, 1))
+        style_features = style_features.view(B, self.in_channels, 1, 1)
+        return style_features
+
+
+class StyleAttentionBlock(nn.Module):
+    """from FTransGAN
+    """
+
+    def __init__(self, in_channels):
+        super().__init__()
+        self.num_local_attention = 3
+        for module_idx in range(1, self.num_local_attention + 1):
+            self.add_module(f"local_attention_{module_idx}",
+                            ContextAwareAttentionBlock(in_channels))
+
+        for module_idx in range(1, self.num_local_attention):
+            self.add_module(f"downsample_{module_idx}",
+                            Conv2d(in_channels, in_channels,
+                                   kernel_size=3, stride=2, padding=1, bias=False))
+
+        self.add_module(f"layer_attention", LayerAttentionBlock(in_channels))
+
+    def forward(self, x, B, K):
+        feature_1 = self.local_attention_1(x)
+
+        x = self.downsample_1(x)
+        feature_2 = self.local_attention_2(x)
+
+        x = self.downsample_2(x)
+        feature_3 = self.local_attention_3(x)
+
+        out = self.layer_attention(x, feature_1, feature_2, feature_3, B, K)
+
+        return out

requirements.txt

@@ -0,0 +1,6 @@
+pytorch-lightning==1.6.0
+omegaconf
+fire
+lpips
+tensorboard
+pillow==8.4.0

trainer.py

@@ -0,0 +1,88 @@
+import argparse
+import glob
+from pathlib import Path
+
+from omegaconf import OmegaConf
+import pytorch_lightning as pl
+from pytorch_lightning.callbacks import ModelCheckpoint
+from pytorch_lightning.loggers import TensorBoardLogger
+
+from lightning import FontLightningModule
+from utils import save_files
+
+
+def load_configuration(path_config):
+    setting = OmegaConf.load(path_config)
+
+    # load hyperparameter
+    hp = OmegaConf.load(setting.config.dataset)
+    hp = OmegaConf.merge(hp, OmegaConf.load(setting.config.model))
+    hp = OmegaConf.merge(hp, OmegaConf.load(setting.config.logging))
+
+    # with lightning setting
+    if hasattr(setting.config, 'lightning'):
+        pl_config = OmegaConf.load(setting.config.lightning)
+        if hasattr(pl_config, 'pl_config'):
+            return hp, pl_config.pl_config
+        return hp, pl_config
+
+    # without lightning setting
+    return hp
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Code to train font style transfer')
+
+    parser.add_argument("--config", type=str, default="./config/setting.yaml",
+                        help="Config file for training")
+    parser.add_argument('-g', '--gpus', type=str, default='0,1',
+                        help="Number of gpus to use (e.g. '0,1,2,3'). Will use all if not given.")
+    parser.add_argument('-p', '--resume_checkpoint_path', type=str, default=None,
+                        help="path of checkpoint for resuming")
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    hp, pl_config = load_configuration(args.config)
+
+    logging_dir = Path(hp.logging.log_dir)
+
+    # call lightning module
+    font_pl = FontLightningModule(hp)
+
+    # set logging
+    hp.logging['log_dir'] = logging_dir / 'tensorboard'
+    savefiles = []
+    for reg in hp.logging.savefiles:
+        savefiles += glob.glob(reg)
+    hp.logging['log_dir'].mkdir(exist_ok=True)
+    save_files(str(logging_dir), savefiles)
+
+    # set tensorboard logger
+    logger = TensorBoardLogger(str(logging_dir), name=str(hp.logging.seed))
+
+    # set checkpoing callback
+    weights_save_path = logging_dir / 'checkpoint' / str(hp.logging.seed)
+    weights_save_path.mkdir(exist_ok=True)
+    checkpoint_callback = ModelCheckpoint(
+        dirpath=str(weights_save_path),
+        **pl_config.checkpoint.callback
+    )
+
+    # set lightning trainer
+    trainer = pl.Trainer(
+        logger=logger,
+        gpus=-1 if args.gpus is None else args.gpus,
+        callbacks=[checkpoint_callback],
+        **pl_config.trainer
+    )
+
+    # let's train
+    trainer.fit(font_pl)
+
+
+if __name__ == "__main__":
+    main()

trainer.sh

@@ -0,0 +1 @@
+python trainer.py --config ./config/setting-google-font.yaml --gpus 0,1,2,3

utils/__init__.py

@@ -0,0 +1,3 @@
+from .logger import *
+from .tb import *
+from .util import *

utils/logger.py

@@ -0,0 +1,39 @@
+import fire
+import logging
+import time
+
+def _custom_logger(name):
+    fmt = '[{}|%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >>> %(message)s'.format(name)
+    fmt_date = '%Y-%m-%d_%T %Z'
+
+    handler = logging.StreamHandler()
+
+    formatter = logging.Formatter(fmt, fmt_date)
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+    logger.addHandler(handler)
+
+def set_logger(logger_name, level):
+    try:
+        time.tzset()
+    except AttributeError as e:
+        print(e)
+        print("Skipping timezone setting.")
+    _custom_logger(name=logger_name)
+    logger = logging.getLogger(logger_name)
+    if level == 'DEBUG':
+        logger.setLevel(logging.DEBUG)
+    elif level == 'INFO':
+        logger.setLevel(logging.INFO)
+    elif level == 'WARNING':
+        logger.setLevel(logging.WARNING)
+    elif level == 'ERROR':
+        logger.setLevel(logging.ERROR)
+    elif level == 'CRITICAL':
+        logger.setLevel(logging.CRITICAL)
+    return logger
+
+if __name__ == '__main__':
+    set_logger("test", "DEBUG")

utils/tb.py

@@ -0,0 +1,25 @@
+import torch
+import numpy as np
+
+def magic_image_handler(img):
+    if isinstance(img, torch.Tensor):
+        img = img.detach().cpu().numpy()
+    if img.ndim == 3:
+        img = img.transpose((1, 2, 0))
+    elif img.ndim == 2:
+        img = np.repeat(img[..., np.newaxis], 3, axis=2)
+    elif img.ndim == 4:
+        img = img[:4]  # first 4 batch
+        img = np.concatenate(img, axis=-1)
+        img = img.transpose((1, 2, 0))
+    elif img.ndim == 5:
+        img = img[:4]  # first 4 batch
+        img = np.concatenate(img, axis=-2)
+        img = np.concatenate(img, axis=-1)
+        img = img.transpose((1, 2, 0))
+    else:
+        raise ValueError(f'img ndim is {img.ndim}, should be 2~4')
+    if img.shape[-1] != 1 or img.shape[-1] != 3:
+        img = np.expand_dims(np.concatenate([img[..., i] for i in range(img.shape[-1])], axis=0), -1)
+    img = np.clip(img, a_min=0, a_max=255)
+    return img

utils/util.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+import shutil
+
+
+def save_files(path_save_, savefiles):
+    path_save = Path(path_save_)
+    path_save.mkdir(exist_ok=True)
+
+    for savefile in savefiles:
+        parents_dir = Path(savefile).parents
+        if len(parents_dir) >= 1:
+            for parent_dir in list(parents_dir)[::-1]:
+                target_dir = path_save / parent_dir
+                target_dir.mkdir(exist_ok=True)
+        try:
+            shutil.copy2(savefile, str(path_save / savefile))
+        except Exception as e:
+            # skip the file
+            print(f'{e} occured while saving {savefile}')
+
+    return  # success
+
+
+if __name__ == "__main__":
+    import glob
+    savefiles = glob.glob('config/*.yaml')
+    savefiles += glob.glob('config/**/*.yaml')
+    save_files(".temp", savefiles)