File size: 8,810 Bytes
4697797
c5cb9ba
 
af6d5f7
4697797
0c738cb
988d509
 
 
4697797
9604304
 
4697797
0c738cb
 
 
 
9604304
 
988d509
4697797
 
 
112fcdf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4697797
988d509
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4697797
 
 
 
 
 
 
af6d5f7
c28f323
af6d5f7
 
 
 
 
 
 
 
c28f323
 
 
 
 
 
 
 
65e5a1d
c28f323
4697797
 
 
 
 
 
 
 
 
c5cb9ba
4697797
 
0c738cb
 
 
988d509
 
 
 
c5cb9ba
65e5a1d
 
 
 
 
 
 
 
 
4697797
 
 
988d509
c5cb9ba
4697797
988d509
c5cb9ba
4697797
988d509
c5cb9ba
4697797
 
 
 
 
 
112fcdf
 
 
 
 
eecdd32
112fcdf
 
 
 
 
 
 
 
312a468
 
4697797
 
 
 
112fcdf
4697797
 
 
 
 
 
 
fc90453
988d509
 
 
 
fc90453
4697797
 
 
 
 
 
c5cb9ba
bbe286f
 
 
 
 
 
 
4697797
 
 
 
 
 
988d509
c5cb9ba
 
 
988d509
c5cb9ba
 
988d509
c5cb9ba
 
988d509
c5cb9ba
 
4697797
 
bbe286f
112fcdf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bbe286f
112fcdf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a8f430e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4697797
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
import gradio as gr
import os
import random
import datetime
from utils import *
from pathlib import Path
import gdown

pre_generate = False

file_url = "https://storage.googleapis.com/derendering_model/derendering_supp.zip"
filename = "derendering_supp.zip"

# Cache videos to speed up demo
video_cache_dir = Path("./cached_videos")
video_cache_dir.mkdir(exist_ok=True)

download_file(file_url, filename)
unzip_file(filename)
print("Downloaded and unzipped the inks.")

diagram = get_svg_content("derendering_supp/derender_diagram.svg")
org = get_svg_content("org/cor.svg")
org_content = f"{org}"

gif_filenames = [
    "christians.gif",
    "good.gif",
    "october.gif",
    "welcome.gif",
    "you.gif",
    "letter.gif",
]
captions = [
    "CHRISTIANS",
    "Good",
    "October",
    "WELOME",
    "you",
    "letter",
]
gif_base64_strings = {
    caption: get_base64_encoded_gif(f"gifs/{name}")
    for caption, name in zip(captions, gif_filenames)
}

sketches = [
    "bird.gif",
    "cat.gif",
    "coffee.gif",
    "penguin.gif",
]
sketches_base64_strings = {
    name: get_base64_encoded_gif(f"sketches/{name}") for name in sketches
}

if not pre_generate:
    print("Downloading pre-generated videos from google drive.")
    # Download from gdown 1oT6zw1EbWg3lavBMXsL28piULGNmqJzA
    gdown.download(
        "https://drive.google.com/uc?id=1oT6zw1EbWg3lavBMXsL28piULGNmqJzA",
        str(video_cache_dir / "gdrive_file.zip"),
        quiet=False,
    )

    # Unzip the file to video_cache_dir
    unzip_file(str(video_cache_dir / "gdrive_file.zip"))
else:
    pregenerate_videos(video_cache_dir=video_cache_dir)
    print("Videos cached.")


def demo(Dataset, Model):
    if Model == "Small-i":
        inkml_path = f"./derendering_supp/small-i_{Dataset}_inkml"
    elif Model == "Small-p":
        inkml_path = f"./derendering_supp/small-p_{Dataset}_inkml"
    elif Model == "Large-i":
        inkml_path = f"./derendering_supp/large-i_{Dataset}_inkml"

    now = datetime.datetime.now()
    random.seed(now.timestamp())
    now = now.strftime("%Y-%m-%d %H:%M:%S")
    print(
        now,
        "Taking sample from dataset:",
        Dataset,
        "and model:",
        Model,
    )
    path = f"./derendering_supp/{Dataset}/images_sample"
    samples = os.listdir(path)
    # Randomly pick a sample
    picked_samples = random.sample(samples, min(1, len(samples)))

    query_modes = ["d+t", "r+d", "vanilla"]
    plot_title = {"r+d": "Recognized: ", "d+t": "OCR Input: ", "vanilla": ""}
    text_outputs = []
    # img_outputs = []
    video_outputs = []
    for name in picked_samples:
        img_path = os.path.join(path, name)
        img = load_and_pad_img_dir(img_path)

        for mode in query_modes:
            example_id = name.strip(".png")
            inkml_file = os.path.join(inkml_path, mode, example_id + ".inkml")
            text_field = parse_inkml_annotations(inkml_file)["textField"]
            output_text = f"{plot_title[mode]}{text_field}"
            text_outputs.append(output_text)
            ink = inkml_to_ink(inkml_file)

            video_filename = f"{Model}_{Dataset}_{mode}_{example_id}.mp4"
            video_filepath = video_cache_dir / video_filename

            if not video_filepath.exists():
                plot_ink_to_video(ink, str(video_filepath), input_image=img)
                print("Cached video at:", video_filepath)
            video_outputs.append("./" + str(video_filepath))

            # fig, ax = plt.subplots()
            # ax.axis("off")
            # plot_ink(ink, ax, input_image=img)
            # buf = BytesIO()
            # fig.savefig(buf, format="png", bbox_inches="tight")
            # plt.close(fig)
            # buf.seek(0)
            # res = Image.open(buf)
            # img_outputs.append(res)
    return (
        img,
        text_outputs[0],
        # img_outputs[0],
        video_outputs[0],
        text_outputs[1],
        # img_outputs[1],
        video_outputs[1],
        text_outputs[2],
        # img_outputs[2],
        video_outputs[2],
    )


with gr.Blocks() as app:
    gr.HTML(org_content)
    gr.Markdown(
        "# InkSight: Offline-to-Online Handwriting Conversion by Learning to Read and Write"
    )
    gr.HTML(
        """
        <div style="display: flex; align-items: center; margin-bottom: 20px;">
            <a href="https://arxiv.org/pdf/2402.05804.pdf" target="_blank" style="font-size: 16px; background-color: #4CAF50; color: white; padding: 5px 7px; text-decoration: none; border-radius: 2px;">
                πŸ“„ Read the Paper
            </a>
        </div>
        """
    )
    gr.HTML(f"<div style='margin: 20px 0;'>{diagram}</div>")
    gr.Markdown(
        """
        πŸš€ This demo highlights the capabilities of Small-i, Small-p, and Large-i across three public datasets (word-level, with 100 random samples each).<br>
        🎲 Select a model variant and dataset (IAM, IMGUR5K, HierText), then hit 'Sample' to view a randomly selected input alongside its corresponding outputs for all three types of inference.<br>
        """
    )
    with gr.Row():
        dataset = gr.Dropdown(
            ["IAM", "IMGUR5K", "HierText"], label="Dataset", value="IAM"
        )
        model = gr.Dropdown(
            ["Small-i", "Large-i", "Small-p"],
            label="InkSight Model Variant",
            value="Small-i",
        )
        im = gr.Image(label="Input Image")

    # with gr.Row():
    #     d_t_img = gr.Image(label="Derender with Text")
    #     r_d_img = gr.Image(label="Recognize and Derender")
    #     vanilla_img = gr.Image(label="Vanilla")

    with gr.Row():
        d_t_text = gr.Textbox(
            label="OCR recognition input to the model", interactive=False
        )
        r_d_text = gr.Textbox(label="Recognition from the model", interactive=False)
        vanilla_text = gr.Textbox(label="Vanilla", interactive=False)
    with gr.Row():
        d_t_vid = gr.Video(
            label="Derender with Text (Click to stop/play)", autoplay=True
        )
        r_d_vid = gr.Video(
            label="Recognize and Derender (Click to stop/play)", autoplay=True
        )
        vanilla_vid = gr.Video(label="Vanilla (Click to stop/play)", autoplay=True)

    with gr.Row():
        btn_sub = gr.Button("Sample")

    btn_sub.click(
        fn=demo,
        inputs=[dataset, model],
        outputs=[
            im,
            d_t_text,
            # d_t_img,
            d_t_vid,
            r_d_text,
            # r_d_img,
            r_d_vid,
            vanilla_text,
            # vanilla_img,
            vanilla_vid,
        ],
    )

    gr.Markdown("## More Word-level Samples")

    html_content = """
    <div style="display: flex; justify-content: space-around; flex-wrap: wrap; gap: 0px;">
    """

    for caption, base64_string in gif_base64_strings.items():
        title = caption
        html_content += f"""
        <div>
            <img src="data:image/gif;base64,{base64_string}" alt="{title}" style="width: 100%; max-width: 200px;">
            <p style="text-align: center;">{title}</p>
        </div>
        """

    html_content += "</div>"

    gr.HTML(html_content)

    # Sketches
    gr.Markdown("## Sketch Samples")

    html_content = """
    <div style="display: flex; justify-content: space-around; flex-wrap: wrap; gap: 0px;">
    """

    for _, base64_string in sketches_base64_strings.items():
        html_content += f"""
        <div>
            <img src="data:image/gif;base64,{base64_string}" style="width: 100%; max-width: 200px;">
        </div>
        """

    html_content += "</div>"

    gr.HTML(html_content)

    gr.Markdown("## Scale Up to Full Page")

    svg1_content = get_svg_content("full_page/danke.svg")
    svg2_content = get_svg_content("full_page/multilingual_demo.svg")
    svg3_content = get_svg_content("full_page/unsplash_frame.svg")

    svg_html_template = """
    <div style="display: block;">
        <div>
            <div style="margin-bottom: 10px;">{}</div>
            <p style="text-align: center;">{}</p> 
        </div>
        <div>
            <div style="margin-bottom: 10px;">{}</div> 
            <p style="text-align: center;">{}</p> 
        </div>
        <div>
            <div style="margin-bottom: 10px;">{}</div> 
            <p style="text-align: center;">{}</p> 
        </div>
    </div>
    """

    full_svg_display = svg_html_template.format(
        svg1_content,
        'Writings on the beach. <a href="https://unsplash.com/photos/text-rG-PerMFjFA">Credit</a>',
        svg2_content,
        "Multilingual handwriting.",
        svg3_content,
        "Handwriting in a frame. <a href='https://unsplash.com/photos/white-wooden-framed-white-board-t7fLWMQl2Lw'>Credit</a>",
    )

    gr.HTML(full_svg_display)


app.launch()