Update app.py

app.py

@@ -7,34 +7,17 @@ import numpy as np
 from sklearn.svm import LinearSVC
 from sklearn import preprocessing
 import pandas as pd
-import kornia
-import torchvision
 
 import random
 import time
 
-from diffusers import LCMScheduler
-from diffusers.models import ImageProjection
-from patch_sdxl import SDEmb
+import replicate
 import torch
-
+import pickle
 
 prompt_list = [p for p in list(set(
                 pd.read_csv('./twitter_prompts.csv').iloc[:, 1].tolist())) if type(p) == str]
 
-
-model_id = "stabilityai/stable-diffusion-xl-base-1.0"
-lcm_lora_id = "latent-consistency/lcm-lora-sdxl"
-
-pipe = SDEmb.from_pretrained(model_id, variant="fp16")
-pipe.load_lora_weights(lcm_lora_id)
-pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
-pipe.to(device=DEVICE, dtype=torch.float16)
-
-pipe.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")
-
-
-
 calibrate_prompts = [
     "4k photo",
     'surrealist art',
@@ -57,20 +40,6 @@ ys = []
 
 start_time = time.time()
 
-output_hidden_state = False if isinstance(pipe.unet.encoder_hid_proj, ImageProjection) else True
-
-
-transform = kornia.augmentation.RandomResizedCrop(size=(224, 224), scale=(.3, .5))
-nom = torchvision.transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
-def patch_encode_image(image):
-    image = torch.tensor(torchvision.transforms.functional.pil_to_tensor(image).to(torch.float16)).repeat(16, 1, 1, 1).to(DEVICE)
-    image = image / 255
-    patches = nom(transform(image))
-    output, _ = pipe.encode_image(
-                patches, DEVICE, 1, output_hidden_state
-            )
-    return output.mean(0, keepdim=True)
-
 
 glob_idx = 0
 
@@ -96,7 +65,6 @@ def next_image():
             pooled_embeds, _ = pipe.encode_image(
                 image[0], DEVICE, 1, output_hidden_state
             )
-            #pooled_embeds = patch_encode_image(image[0])
 
             embs.append(pooled_embeds)
             return image[0]
@@ -131,19 +99,10 @@ def next_image():
             prompt= 'an image' if glob_idx % 2 == 0 else rng_prompt
             print(prompt)
 
-            image = pipe(
-            prompt=prompt,
-            ip_adapter_emb=im_emb,
-            height=1024,
-            width=1024,
-            num_inference_steps=8,
-            guidance_scale=0,
-            ).images
-
-            im_emb, _ = pipe.encode_image(
-                image[0], DEVICE, 1, output_hidden_state
+            image, im_emb = replicate.run(
+            "rynmurdock/zahir:43177e0594f3bc2e3560170ff0ffb6d1cacdddda1be25fbcd4348ef02b0b7d0f",
+            input={"prompt": prompt, 'im_emg': pickle.dumps(im_emb)}
             )
-            #im_emb = patch_encode_image(image[0])
 
             embs.append(im_emb)