support new year feature

.gitattributes

@@ -1 +1,2 @@
 __assets__/image_animation/zhening/zhening.jpeg filter=lfs diff=lfs merge=lfs -text
+zyhzx.ttf filter=lfs diff=lfs merge=lfs -text

app-huggingface.py

@@ -1,5 +1,5 @@
-import json
 import copy
+import json
 import os
 import os.path as osp
 import random
@@ -7,17 +7,54 @@ from argparse import ArgumentParser
 from datetime import datetime
 
 import gradio as gr
+import moviepy.editor as mpy
 import numpy as np
 import torch
 from huggingface_hub import hf_hub_download
 from omegaconf import OmegaConf
-from PIL import Image
+from PIL import Image, ImageDraw, ImageFont
 
 from animatediff.pipelines import I2VPipeline
 from animatediff.utils.util import RANGE_LIST, save_videos_grid
 
 sample_idx = 0
 
+
+def convert_gif_to_mp4(gif_path, mp4_path):
+
+    clip = mpy.VideoFileClip(gif_path)
+    clip.write_videofile(mp4_path)
+
+
+def add_text(gif_file: str, mp4_file: str, text_index: int):
+    image = Image.open(gif_file)
+
+    frames = []
+    try:
+        while True:
+            frames.append(image.copy().convert('RGB'))
+            image.seek(len(frames))
+    except EOFError:
+        pass
+
+    text = ['Happy New Year!'][text_index]
+    size = [36, 36, 36, 36][text_index]
+
+    for i, frame in enumerate(frames):
+        font = ImageFont.truetype('zyhzx.ttf', size=size + i * 2)
+        draw = ImageDraw.Draw(frame)
+
+        text_width, text_height = draw.textsize(text, font=font)
+        image_width, image_height = image.size
+        x = (image_width - text_width) // 2
+        y = (image_height - text_height) - (image_height - text_height) // 8
+
+        draw.text((x, y), text, fill='red', font=font)
+
+    frames[0].save(gif_file, save_all=True, append_images=frames[1:], loop=0)
+    mp4_file = convert_gif_to_mp4(gif_file, mp4_file)
+    return gif_file, mp4_file
+
 css = """
 .toolbutton {
     margin-buttom: 0em 0em 0em 0em;
@@ -43,7 +80,6 @@ LOCAL_DEBUG = args.local_debug
 BASE_CONFIG = 'example/config/base.yaml'
 STYLE_CONFIG_LIST = {
     '3d_cartoon': './example/openxlab/3-3d.yaml',
-    # 'realistic': './example/openxlab/1-realistic.yaml',
 }
 
 
@@ -79,11 +115,7 @@ if not LOCAL_DEBUG:
     PIA_PATH = hf_hub_download(repo_id='Leoxing/PIA',
                                filename='pia.ckpt', cache_dir=PIA_PATH)
     PIA_PATH = '/'.join(PIA_PATH.split('/')[:-1])
-    # os.system('bash download_bashscripts/1-RealisticVision.sh')
     os.system('bash download_bashscripts/2-RcnzCartoon.sh')
-    # hf_hub_download(repo_id='frankjoshua/realisticVisionV51_v51VAE',
-    #                 filename='realisticVisionV51_v51VAE.safetensors',
-    #                 cache_dir='')
     print(os.listdir(DreamBooth_LoRA_PATH))
 
     # unet
@@ -186,7 +218,6 @@ class AnimateController:
         self.pipeline_dict = self.load_model_list()
 
     def load_model_list(self):
-        print(os.listdir(DreamBooth_LoRA_PATH))
         pipeline_dict = dict()
         for style, cfg in self.style_configs.items():
             dreambooth_path = cfg.get('dreambooth', 'none')
@@ -235,9 +266,13 @@ class AnimateController:
         ip_adapter_scale,
         style,
         example_img=None,
+        with_text=False,
+        text_idx=0,
         progress=gr.Progress(),
     ):
 
+        global sample_idx
+
         if init_img is None:
             if example_img is None:
                 gr.Warning('Please upload image or use example images.')
@@ -247,18 +282,24 @@ class AnimateController:
 
         if seed_textbox != -1 and seed_textbox != "":
             torch.manual_seed(int(seed_textbox))
+            seed = int(seed_textbox)
         else:
             seed = torch.initial_seed()
-
+        generator = torch.Generator(device='cuda')
+        generator.manual_seed(seed)
         seed_everything(seed)
 
+        print(f'Seed: {seed}')
+
         pipeline = self.pipeline_dict[style]
         init_img, h, w = preprocess_img(init_img)
         print(f'img size: {h, w}')
+
         sample = pipeline(
             image=init_img,
             prompt=prompt_textbox,
             negative_prompt=negative_prompt_textbox,
+            generator=generator,
             num_inference_steps=sample_step_slider,
             guidance_scale=cfg_scale_slider,
             width=w,
@@ -269,9 +310,13 @@ class AnimateController:
             progress_fn=progress,
         ).videos
 
-        save_sample_path = os.path.join(
+        save_sample_path_mp4 = os.path.join(
             self.savedir_sample, f"{sample_idx}.mp4")
-        save_videos_grid(sample, save_sample_path)
+        save_sample_path_gif = os.path.join(
+            self.savedir_sample, f"{sample_idx}.gif")
+
+        save_videos_grid(sample, save_sample_path_mp4)
+        save_videos_grid(sample, save_sample_path_gif)
 
         sample_config = {
             "prompt": prompt_textbox,
@@ -283,13 +328,21 @@ class AnimateController:
             "seed": seed,
             "motion": motion_scale,
         }
+        print(sample_config)
         json_str = json.dumps(sample_config, indent=4)
         with open(os.path.join(self.savedir, "logs.json"), "a") as f:
             f.write(json_str)
             f.write("\n\n")
 
-        # return init_img_bk, save_sample_path
-        return save_sample_path
+        if with_text:
+            add_text(save_sample_path_gif, save_sample_path_mp4, text_idx)
+
+        sample_idx += 1
+        # return (save_sample_path_mp4,
+        #         gr.update(link=f'file/{save_sample_path_mp4}'),
+        #         gr.update(link=f'file/{save_sample_path_gif}'))
+        return (save_sample_path_mp4,
+                [save_sample_path_mp4, save_sample_path_gif])
 
 
 controller = AnimateController()
@@ -297,6 +350,26 @@ controller = AnimateController()
 
 def ui():
     with gr.Blocks(css=css) as demo:
+        # build state for default buttons
+        example_img = gr.State(
+            value='__assets__/image_animation/zhening/zhening.jpeg')
+        default_motion = gr.State(value=1)
+        default_prompt1 = gr.State(
+            value='lift a red envelope, Chinese new year')
+        default_prompt2 = gr.State(
+            value='smiling, Chinese costume, Chinese new year')
+        default_prompt3 = gr.State(
+            value='angry, Chinese costume, Chinese new year')
+        default_prompt4 = gr.State(value='sparklers, Chinese new year')
+        default_n_prompt = gr.State(value='wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg')
+        default_seed = gr.State(10201304011203481448)
+        default_ip_adapter_scale = gr.State(0.2)
+        default_style = gr.State('3d_cartoon')
+        default_cfg = gr.State(7.5)
+        default_1_idx = gr.State(0)
+        default_2_idx = gr.State(1)
+        default_3_idx = gr.State(2)
+        default_4_idx = gr.State(3)
 
         gr.HTML(
             "<div align='center'><font size='7'> <img src=\"file/pia.png\" style=\"height: 72px;\"/ > Your Personalized Image Animator</font></div>"
@@ -311,33 +384,27 @@ def ui():
             )
 
         with gr.Row(equal_height=False):
-            # build state for default buttons
-            example_img = gr.State(value='__assets__/image_animation/zhening/zhening.jpeg')
-            default_motion = gr.State(value=1)
-            default_prompt1 = gr.State(value='lift a red envelope, Chinese new year')
-            default_prompt2 = gr.State(value='smiling, Chinese costume, Chinese new year')
-            default_prompt3 = gr.State(value='angry, Chinese costume, Chinese new year')
-            default_prompt4 = gr.State(value='sparklers, Chinese new year')
-            default_n_prompt = gr.State(value='wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg')
-            default_seed = gr.State(10201304011203481448)
-            default_ip_adapter_scale = gr.State(0.2)
-            default_style = gr.State('3d_cartoon')
-            default_cfg = gr.State(7.5)
-
             with gr.Column():
                 with gr.Row():
                     init_img = gr.Image(label='Input Image')
 
                 gr.Markdown('## Fast Try!')
                 with gr.Row():
-                    with gr.Column(scale=0.125):
-                        default_1 = gr.Button('🧧', variant='primary')
-                    with gr.Column(scale=0.125):
-                        default_2 = gr.Button('🤗', variant='primary')
-                    with gr.Column(scale=0.125):
-                        default_3 = gr.Button('😡', variant='primary')
-                    with gr.Column(scale=0.125):
-                        default_4 = gr.Button('🧨', variant='primary')
+                    with gr.Column(scale=1, min_width=50):
+                        default_1 = gr.Button(
+                            '🧧', variant='primary', size='sm')
+                    with gr.Column(scale=1, min_width=50):
+                        default_2 = gr.Button(
+                            '🤗', variant='primary', size='sm')
+                    with gr.Column(scale=1, min_width=50):
+                        default_3 = gr.Button(
+                            '😡', variant='primary', size='sm')
+                    with gr.Column(scale=1, min_width=50):
+                        default_4 = gr.Button(
+                            '🧨', variant='primary', size='sm')
+
+                    with gr.Column(scale=1.5, min_width=150):
+                        with_wishes = gr.Checkbox(label='With Wishes✨')
 
                 style_dropdown = gr.Dropdown(label='Style', choices=list(
                     STYLE_CONFIG_LIST.keys()), value=list(STYLE_CONFIG_LIST.keys())[0])
@@ -374,14 +441,17 @@ def ui():
                         value=controller.fetch_default_n_prompt(
                             list(STYLE_CONFIG_LIST.keys())[0])[0],
                         label="Negative prompt", lines=2)
-                    ip_adapter_scale = gr.Slider(
-                        label='IP-Apdater Scale', value=controller.fetch_default_n_prompt(
-                            list(STYLE_CONFIG_LIST.keys())[0])[1], minimum=0, maximum=1)
+
                     sample_step_slider = gr.Slider(
                         label="Sampling steps", value=20, minimum=10, maximum=100, step=1)
 
                     cfg_scale_slider = gr.Slider(
                         label="CFG Scale", value=7.5, minimum=0, maximum=20)
+                    ip_adapter_scale = gr.Slider(
+                        label='IP-Apdater Scale',
+                        value=controller.fetch_default_n_prompt(
+                            list(STYLE_CONFIG_LIST.keys())[0])[1],
+                        minimum=0, maximum=1)
 
                     with gr.Row():
                         seed_textbox = gr.Textbox(label="Seed", value=-1)
@@ -396,8 +466,11 @@ def ui():
                 generate_button = gr.Button(
                     value="Generate", variant='primary')
 
-            result_video = gr.Video(
-                label="Generated Animation", interactive=False)
+            with gr.Column():
+                result_video = gr.Video(
+                    label="Generated Animation", interactive=False)
+                with gr.Row():
+                    download = gr.Files(file_types=['gif', 'mp4'], label='Donwload Output')
 
         style_dropdown.change(fn=controller.fetch_default_n_prompt,
                               inputs=[style_dropdown],
@@ -417,9 +490,10 @@ def ui():
                 seed_textbox,
                 ip_adapter_scale,
                 style_dropdown,
+                # None,
             ],
-            outputs=[result_video]
-        )
+            outputs=[result_video, download])
+
         default_1.click(
             fn=controller.animate,
             inputs=[
@@ -428,13 +502,15 @@ def ui():
                 default_prompt1,
                 default_n_prompt,
                 sample_step_slider,
-                default_cfg ,
+                default_cfg,
                 default_seed,
                 default_ip_adapter_scale,
                 default_style,
                 example_img,
+                with_wishes,
+                default_1_idx,
             ],
-            outputs=[result_video])
+            outputs=[result_video, download])
         default_2.click(
             fn=controller.animate,
             inputs=[
@@ -443,13 +519,15 @@ def ui():
                 default_prompt2,
                 default_n_prompt,
                 sample_step_slider,
-                default_cfg ,
+                default_cfg,
                 default_seed,
                 default_ip_adapter_scale,
                 default_style,
                 example_img,
+                with_wishes,
+                default_2_idx,
             ],
-            outputs=[result_video])
+            outputs=[result_video, download])
         default_3.click(
             fn=controller.animate,
             inputs=[
@@ -458,13 +536,15 @@ def ui():
                 default_prompt3,
                 default_n_prompt,
                 sample_step_slider,
-                default_cfg ,
+                default_cfg,
                 default_seed,
                 default_ip_adapter_scale,
                 default_style,
                 example_img,
+                with_wishes,
+                default_3_idx,
             ],
-            outputs=[result_video])
+            outputs=[result_video, download])
         default_4.click(
             fn=controller.animate,
             inputs=[
@@ -473,13 +553,21 @@ def ui():
                 default_prompt4,
                 default_n_prompt,
                 sample_step_slider,
-                default_cfg ,
+                default_cfg,
                 default_seed,
                 default_ip_adapter_scale,
                 default_style,
                 example_img,
+                with_wishes,
+                default_4_idx,
             ],
-            outputs=[result_video])
+            outputs=[result_video, download])
+
+        # save_as_gif.click(
+        #     fn=None,
+        #     inputs=[save_as_gif],
+        #     js=_js_open_if_not_logged_in,
+        # )
 
         def create_example(input_list):
             return gr.Examples(
@@ -500,16 +588,16 @@ def ui():
         create_example(
             [
                 [
-                    './__assets__/image_animation/yiming/yiming.jpeg',
-                    './__assets__/image_animation/yiming/yiming.mp4',
+                    '__assets__/image_animation/yiming/yiming.jpeg',
+                    '__assets__/image_animation/yiming/yiming.mp4',
                     '1boy in Christmas suit, lift a Christmas gift',
                     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                     '3d_cartoon',
                     2,
                 ],
                 [
-                    './__assets__/image_animation/yanhong/yanhong.png',
-                    './__assets__/image_animation/yanhong/yanhong.mp4',
+                    '__assets__/image_animation/yanhong/yanhong.png',
+                    '__assets__/image_animation/yanhong/yanhong.mp4',
                     '1girl lift a Christmas gift',
                     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                     '3d_cartoon',
@@ -523,16 +611,16 @@ def ui():
             create_example([
                 [
 
-                    './__assets__/image_animation/style_transfer/anya/anya.jpg',
-                    './__assets__/image_animation/style_transfer/anya/2.mp4',
+                    '__assets__/image_animation/style_transfer/anya/anya.jpg',
+                    '__assets__/image_animation/style_transfer/anya/2.mp4',
                     '1girl open mouth ',
                     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                     '3d_cartoon',
                     3,
                 ],
                 [
-                    './__assets__/image_animation/magnitude/genshin/genshin.jpg',
-                    './__assets__/image_animation/magnitude/genshin/3.mp4',
+                    '__assets__/image_animation/magnitude/genshin/genshin.jpg',
+                    '__assets__/image_animation/magnitude/genshin/3.mp4',
                     'cherry blossoms in the wind, raidenshogundef, yaemikodef, best quality, 4k',
                     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                     '3d_cartoon',
@@ -544,49 +632,25 @@ def ui():
         with gr.Accordion('More Examples for Prompt Changing', open=False):
             create_example(
                 [
-                    # [
-                    #     './__assets__/image_animation/real/lighthouse.jpg',
-                    #     './__assets__/image_animation/real/1.mp4',
-                    #     'lightning, lighthouse',
-                    #     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
-                    #     'realistic',
-                    #     1,
-                    # ],
-                    # [
-                    #     './__assets__/image_animation/real/lighthouse.jpg',
-                    #     './__assets__/image_animation/real/2.mp4',
-                    #     'sun rising, lighthouse',
-                    #     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
-                    #     'realistic',
-                    #     1,
-                    # ],
-                    # [
-                    #     './__assets__/image_animation/real/lighthouse.jpg',
-                    #     './__assets__/image_animation/real/3.mp4',
-                    #     'fireworks, lighthouse',
-                    #     'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
-                    #     'realistic',
-                    #     1,
-                    # ],
                     [
-                        './__assets__/image_animation/rcnz/harry.png',
-                        './__assets__/image_animation/rcnz/1.mp4',
+                        '__assets__/image_animation/rcnz/harry.png',
+                        '__assets__/image_animation/rcnz/1.mp4',
                         '1boy smiling',
                         'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                         '3d_cartoon',
                         2
                     ],
                     [
-                        './__assets__/image_animation/rcnz/harry.png',
-                        './__assets__/image_animation/rcnz/2.mp4',
+                        '__assets__/image_animation/rcnz/harry.png',
+                        '__assets__/image_animation/rcnz/2.mp4',
                         '1boy playing magic fire',
                         'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                         '3d_cartoon',
                         2
                     ],
                     [
-                        './__assets__/image_animation/rcnz/harry.png',
-                        './__assets__/image_animation/rcnz/3.mp4',
+                        '__assets__/image_animation/rcnz/harry.png',
+                        '__assets__/image_animation/rcnz/3.mp4',
                         '1boy is waving hands',
                         'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                         '3d_cartoon',
@@ -598,24 +662,24 @@ def ui():
                 create_example(
                     [
                         [
-                            './__assets__/image_animation/magnitude/labrador.png',
-                            './__assets__/image_animation/magnitude/1.mp4',
+                            '__assets__/image_animation/magnitude/labrador.png',
+                            '__assets__/image_animation/magnitude/1.mp4',
                             'cherry blossoms in the wind, raidenshogundef, yaemikodef, best quality, 4k',
                             'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                             '3d_cartoon',
                             1,
                         ],
                         [
-                            './__assets__/image_animation/magnitude/labrador.png',
-                            './__assets__/image_animation/magnitude/2.mp4',
+                            '__assets__/image_animation/magnitude/labrador.png',
+                            '__assets__/image_animation/magnitude/2.mp4',
                             'cherry blossoms in the wind, raidenshogundef, yaemikodef, best quality, 4k',
                             'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                             '3d_cartoon',
                             2,
                         ],
                         [
-                            './__assets__/image_animation/magnitude/labrador.png',
-                            './__assets__/image_animation/magnitude/3.mp4',
+                            '__assets__/image_animation/magnitude/labrador.png',
+                            '__assets__/image_animation/magnitude/3.mp4',
                             'cherry blossoms in the wind, raidenshogundef, yaemikodef, best quality, 4k',
                             'wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg',
                             '3d_cartoon',
@@ -632,4 +696,4 @@ if __name__ == "__main__":
     demo.launch(server_name=args.server_name,
                 server_port=args.port, share=args.share,
                 max_threads=40,
-                allowed_paths=['pia.png'])
+                allowed_paths=['pia.png', 'samples'])

requirements.txt

@@ -11,3 +11,4 @@ safetensors
 gradio==4.7.1
 moviepy
 pytorch_lightning==1.9.4
+pillow==9.5.0

zyhzx.ttf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e787d88450c8d8c6ba84ffd7b1d8105708270165fdce8b365656bca0294e54
+size 15260048