yo man

app.py

@@ -70,10 +70,18 @@ def tv_loss(input):
     x_diff = input[..., :-1, 1:] - input[..., :-1, :-1]
     y_diff = input[..., 1:, :-1] - input[..., :-1, :-1]
     return (x_diff**2 + y_diff**2).mean([1, 2, 3])
+
+def l1_loss(input):
+    """L1 total variation loss, as in Mahendran et al."""
+    input = F.pad(input, (0, 1, 0, 1), 'replicate')
+    x_diff = input[..., :-1, 1:] - input[..., :-1, :-1]
+    y_diff = input[..., 1:, :-1] - input[..., :-1, :-1]
+    return (torch.abs(x_diff**1) + torch.abs(y_diff**1)).mean([1, 2, 3])
+
 def range_loss(input):
     return (input - input.clamp(-1, 1)).pow(2).mean([1, 2, 3])
     
-def inference(text, init_image, skip_timesteps, clip_guidance_scale, tv_scale, range_scale, init_scale, seed, image_prompts,timestep_respacing, cutn):
+def inference(text, init_image, skip_timesteps, clip_guidance_scale, tv_scale, l1_scale, range_scale, init_scale, seed, image_prompts,timestep_respacing, cutn):
     # Model settings
     model_config = model_and_diffusion_defaults()
     model_config.update({
@@ -120,6 +128,7 @@ def inference(text, init_image, skip_timesteps, clip_guidance_scale, tv_scale, r
     batch_size = 1
     clip_guidance_scale = clip_guidance_scale  # Controls how much the image should look like the prompt.
     tv_scale = tv_scale             # Controls the smoothness of the final output.
+    l1_scale = l1_scale
     range_scale = range_scale            # Controls how far out of range RGB values are allowed to be.
     cutn = cutn
     n_batches = 1
@@ -160,26 +169,24 @@ def inference(text, init_image, skip_timesteps, clip_guidance_scale, tv_scale, r
         init = init.resize((side_x, side_y), Image.LANCZOS)
         init = TF.to_tensor(init).to(device).unsqueeze(0).mul(2).sub(1)
     cur_t = None
-    def cond_fn(x, t, y=None):
-        with torch.enable_grad():
-            x = x.detach().requires_grad_()
-            n = x.shape[0]
-            my_t = torch.ones([n], device=device, dtype=torch.long) * cur_t
-            out = diffusion.p_mean_variance(model, x, my_t, clip_denoised=False, model_kwargs={'y': y})
-            fac = diffusion.sqrt_one_minus_alphas_cumprod[cur_t]
-            x_in = out['pred_xstart'] * fac + x * (1 - fac)
-            clip_in = normalize(make_cutouts(x_in.add(1).div(2)))
-            image_embeds = clip_model.encode_image(clip_in).float()
-            dists = spherical_dist_loss(image_embeds.unsqueeze(1), target_embeds.unsqueeze(0))
-            dists = dists.view([cutn, n, -1])
-            losses = dists.mul(weights).sum(2).mean(0)
-            tv_losses = tv_loss(x_in)
-            range_losses = range_loss(out['pred_xstart'])
-            loss = losses.sum() * clip_guidance_scale + tv_losses.sum() * tv_scale + range_losses.sum() * range_scale
-            if init is not None and init_scale:
-                init_losses = lpips_model(x_in, init)
-                loss = loss + init_losses.sum() * init_scale
-            return -torch.autograd.grad(loss, x)[0]
+
+    def cond_fn(x, t, out, y=None):
+        n = x.shape[0]
+        fac = diffusion.sqrt_one_minus_alphas_cumprod[cur_t]
+        x_in = out['pred_xstart'] * fac + x * (1 - fac)
+        clip_in = normalize(make_cutouts(x_in.add(1).div(2)))
+        image_embeds = clip_model.encode_image(clip_in).float()
+        dists = spherical_dist_loss(image_embeds.unsqueeze(1), target_embeds.unsqueeze(0))
+        dists = dists.view([cutn, n, -1])
+        losses = dists.mul(weights).sum(2).mean(0)
+        tv_losses = tv_loss(x_in)
+        range_losses = range_loss(out['pred_xstart'])
+        l1_losses = l1_loss(x_in)
+        loss = losses.sum() * clip_guidance_scale + tv_losses.sum() * tv_scale + range_losses.sum() * range_scale + l1_losses.sum() * l1_scale
+        if init is not None and init_scale:
+            init_losses = lpips_model(x_in, init)
+            loss = loss + init_losses.sum() * init_scale
+        return -torch.autograd.grad(loss, x)[0]
     if model_config['timestep_respacing'].startswith('ddim'):
         sample_fn = diffusion.ddim_sample_loop_progressive
     else:
@@ -212,11 +219,10 @@ def inference(text, init_image, skip_timesteps, clip_guidance_scale, tv_scale, r
         writer.append_data(np.array(im))
     writer.close()
     return img, 'video.mp4'
-
-
+    
 title = "CLIP Guided Diffusion HQ"
 description = "Gradio demo for CLIP Guided Diffusion. To use it, simply add your text, or click one of the examples to load them. Read more at the links below."
 article = "<p style='text-align: center'> By Katherine Crowson (https://github.com/crowsonkb, https://twitter.com/RiversHaveWings). It uses OpenAI's 256x256 unconditional ImageNet diffusion model (https://github.com/openai/guided-diffusion) together with CLIP (https://github.com/openai/CLIP) to connect text prompts with images. | <a href='https://colab.research.google.com/drive/12a_Wrfi2_gwwAuN3VvMTwVMz9TfqctNj' target='_blank'>Colab</a></p>"
-iface = gr.Interface(inference, inputs=["text",gr.inputs.Image(type="file", label='initial image (optional)', optional=True),gr.inputs.Slider(minimum=0, maximum=45, step=1, default=10, label="skip_timesteps"), gr.inputs.Slider(minimum=0, maximum=3000, step=1, default=600, label="clip guidance scale (Controls how much the image should look like the prompt)"), gr.inputs.Slider(minimum=0, maximum=1000, step=1, default=0, label="tv_scale (Controls the smoothness of the final output)"), gr.inputs.Slider(minimum=0, maximum=1000, step=1, default=0, label="range_scale (Controls how far out of range RGB values are allowed to be)"), gr.inputs.Slider(minimum=0, maximum=1000, step=1, default=0, label="init_scale (This enhances the effect of the init image)"), gr.inputs.Number(default=0, label="Seed"), gr.inputs.Image(type="file", label='image prompt (optional)', optional=True), gr.inputs.Slider(minimum=50, maximum=500, step=1, default=50, label="timestep respacing"),gr.inputs.Slider(minimum=1, maximum=64, step=1, default=32, label="cutn")], outputs=["image","video"], title=title, description=description, article=article, examples=[["coral reef city by artistation artists", None, 0, 1000, 150, 50, 0, 0, None, 90, 32]],
+iface = gr.Interface(inference, inputs=["text",gr.inputs.Image(type="file", label='initial image (optional)', optional=True),gr.inputs.Slider(minimum=0, maximum=500, step=1, default=10, label="skip_timesteps"), gr.inputs.Slider(minimum=0, maximum=3000, step=1, default=600, label="clip guidance scale (Controls how much the image should look like the prompt)"), gr.inputs.Slider(minimum=0, maximum=1000, step=1, default=0, label="tv_scale (Controls the smoothness of the final output)"),gr.inputs.Slider(minimum=0, maximum=500, step=1, default=0, label="l1_scale (How much to punish for straying from init_image)"), gr.inputs.Slider(minimum=0, maximum=1000, step=1, default=0, label="range_scale (Controls how far out of range RGB values are allowed to be)"), gr.inputs.Slider(minimum=0, maximum=1000, step=1, default=0, label="init_scale (This enhances the effect of the init image)"), gr.inputs.Number(default=0, label="Seed"), gr.inputs.Image(type="file", label='image prompt (optional)', optional=True), gr.inputs.Slider(minimum=50, maximum=500, step=1, default=50, label="timestep respacing"),gr.inputs.Slider(minimum=1, maximum=64, step=1, default=32, label="cutn")], outputs=["image","video"], title=title, description=description, article=article, examples=[["coral reef city by artistation artists", None, 0, 1000, 150, 50, 0, 0, None, 90, 32]],
     enable_queue=True)
 iface.launch()