File size: 7,985 Bytes
31f45e8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26d7dfc
31f45e8
 
 
 
 
 
 
debc5f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31f45e8
debc5f3
 
 
 
31f45e8
debc5f3
 
 
 
31f45e8
debc5f3
 
 
 
 
 
 
 
 
31f45e8
 
debc5f3
31f45e8
 
debc5f3
 
 
 
 
31f45e8
debc5f3
 
 
31f45e8
debc5f3
 
 
31f45e8
debc5f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31f45e8
debc5f3
31f45e8
debc5f3
 
 
 
 
 
31f45e8
debc5f3
31f45e8
debc5f3
31f45e8
debc5f3
 
 
 
 
31f45e8
 
 
debc5f3
 
 
 
 
 
 
 
31f45e8
 
debc5f3
 
 
 
31f45e8
 
debc5f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
from inpaint_zoom.utils.zoom_in_utils import image_grid, shrink_and_paste_on_blank, dummy, write_video
from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
from PIL import Image
import gradio as gr
import numpy as np
import torch
import os
os.environ["CUDA_VISIBLE_DEVICES"]="0"


stable_paint_model_list = [
  "stabilityai/stable-diffusion-2-inpainting", 
  "runwayml/stable-diffusion-inpainting"
]

stable_paint_prompt_list = [
        "children running in the forest , sunny, bright, by studio ghibli painting, superior quality, masterpiece,  traditional Japanese colors, by Grzegorz Rutkowski, concept art",
        "A beautiful landscape of a mountain range with a lake in the foreground",
]

stable_paint_negative_prompt_list = [
        "lurry, bad art, blurred, text, watermark",
    ]

class StableDiffusionZoomIn:
    def __init__(self):
        self.pipe = None
        
    def load_model(self, model_id):
        if self.pipe is None:
            self.pipe = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, revision="fp16")
            
        self.pipe.scheduler = DPMSolverMultistepScheduler.from_config(self.pipe.scheduler.config)
        self.pipe = self.pipe.to("cuda")

        self.pipe.safety_checker = dummy
        self.pipe.enable_attention_slicing() 
        self.pipe.enable_xformers_memory_efficient_attention()
        self.g_cuda = torch.Generator(device='cuda')
            
        return self.pipe

    def generate_video(
        self,
        model_id,
        prompt,
        negative_prompt,
        guidance_scale,
        num_inference_steps,
        ):
        
        pipe = self.load_model(model_id)

        num_init_images = 2
        seed = 9999 
        height = 512
        width = height 

        current_image = Image.new(mode="RGBA", size=(height, width))
        mask_image = np.array(current_image)[:,:,3] 
        mask_image = Image.fromarray(255-mask_image).convert("RGB")
        current_image = current_image.convert("RGB")

        init_images =  pipe(prompt=[prompt]*num_init_images,
                            negative_prompt=[negative_prompt]*num_init_images,
                            image=current_image,
                            guidance_scale = guidance_scale,
                            height = height,
                            width = width, 
                            generator = self.g_cuda.manual_seed(seed),
                            mask_image=mask_image, 
                            num_inference_steps=num_inference_steps)[0]


        image_grid(init_images, rows=1, cols=num_init_images)


        init_image_selected = 1 #@param
        if num_init_images == 1:
            init_image_selected = 0
        else:
            init_image_selected = init_image_selected - 1

        num_outpainting_steps = 20 #@param
        mask_width = 128 #@param
        num_interpol_frames = 30 #@param 

        current_image = init_images[init_image_selected]
        all_frames = []
        all_frames.append(current_image)

        for i in range(num_outpainting_steps):
            print('Generating image: ' + str(i+1) + ' / ' + str(num_outpainting_steps))

            prev_image_fix = current_image

            prev_image = shrink_and_paste_on_blank(current_image, mask_width)

            current_image = prev_image

            #create mask (black image with white mask_width width edges)
            mask_image = np.array(current_image)[:,:,3] 
            mask_image = Image.fromarray(255-mask_image).convert("RGB")

            #inpainting step
            current_image = current_image.convert("RGB")
            images = pipe(prompt=prompt,
                            negative_prompt=negative_prompt,
                            image=current_image,
                            guidance_scale = guidance_scale,
                            height = height,
                            width = width, 
                            #this can make the whole thing deterministic but the output less exciting
                            #generator = g_cuda.manual_seed(seed), 
                            mask_image=mask_image, 
                            num_inference_steps=num_inference_steps)[0]
            current_image = images[0]
            current_image.paste(prev_image, mask=prev_image)

            #interpolation steps bewteen 2 inpainted images (=sequential zoom and crop)
            for j in range(num_interpol_frames - 1):
                interpol_image = current_image
                interpol_width = round(
                    (1- ( 1-2*mask_width/height )**( 1-(j+1)/num_interpol_frames ) )*height/2 
                    )
                interpol_image = interpol_image.crop((interpol_width,
                                                    interpol_width,
                                                    width - interpol_width,
                                                    height - interpol_width))

                interpol_image = interpol_image.resize((height, width))

                #paste the higher resolution previous image in the middle to avoid drop in quality caused by zooming
                interpol_width2 = round(
                    ( 1 - (height-2*mask_width) / (height-2*interpol_width) ) / 2*height
                    )
                prev_image_fix_crop = shrink_and_paste_on_blank(prev_image_fix, interpol_width2)
                interpol_image.paste(prev_image_fix_crop, mask = prev_image_fix_crop)

                all_frames.append(interpol_image)

            all_frames.append(current_image)

        video_file_name = "infinite_zoom_out"
        fps = 30 
        save_path = video_file_name + ".mp4"
        write_video(save_path, all_frames, fps)
        return save_path


  
    def app():
        with gr.Blocks():
            with gr.Row():
                with gr.Column():
                    text2image_in_model_path = gr.Dropdown(
                        choices=stable_paint_model_list, 
                        value=stable_paint_model_list[0], 
                        label='Text-Image Model Id'
                    )

                    text2image_in_prompt = gr.Textbox(
                        lines=1, 
                        value=stable_paint_prompt_list[0], 
                        label='Prompt'
                    )

                    text2image_in_negative_prompt = gr.Textbox(
                        lines=1, 
                        value=stable_paint_negative_prompt_list[0], 
                        label='Negative Prompt'
                    )

                    with gr.Row():
                        with gr.Column():
                            text2image_in_guidance_scale = gr.Slider(
                                minimum=0.1, 
                                maximum=15, 
                                step=0.1, 
                                value=7.5, 
                                label='Guidance Scale'
                            )

                            text2image_in_num_inference_step = gr.Slider(
                                minimum=1, 
                                maximum=100, 
                                step=1, 
                                value=50, 
                                label='Num Inference Step'
                            )

                    text2image_in_predict = gr.Button(value='Generator')
            
                with gr.Column():
                    output_image = gr.Video(label='Output')
                        
            
            text2image_in_predict.click(
                fn=StableDiffusionZoomIn().generate_video,
                inputs=[
                    text2image_in_model_path,
                    text2image_in_prompt,
                    text2image_in_negative_prompt,
                    text2image_in_guidance_scale,
                    text2image_in_num_inference_step,
                ],
                outputs=output_image
            )