lfs and sync with blue-tigers github

.gitattributes

@@ -1 +1,21 @@
 nsfweffnetv2-b02-3epochs.h5 filter=lfs diff=lfs merge=lfs -text
+fifth.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+ninth.im_.pt filter=lfs diff=lfs merge=lfs -text
+tenth.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+third.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+eigth.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+first.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+fourth.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+ninth.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+sixth.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+tenth.im_.pt filter=lfs diff=lfs merge=lfs -text
+eigth.im_.pt filter=lfs diff=lfs merge=lfs -text
+seventh.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+sixth.im_.pt filter=lfs diff=lfs merge=lfs -text
+third.im_.pt filter=lfs diff=lfs merge=lfs -text
+fifth.im_.pt filter=lfs diff=lfs merge=lfs -text
+first.im_.pt filter=lfs diff=lfs merge=lfs -text
+fourth.im_.pt filter=lfs diff=lfs merge=lfs -text
+second.gemb_.pt filter=lfs diff=lfs merge=lfs -text
+second.im_.pt filter=lfs diff=lfs merge=lfs -text
+seventh.im_.pt filter=lfs diff=lfs merge=lfs -text

app.py

@@ -10,12 +10,9 @@ STEPS = 6
 output_hidden_state = False
 device = "cuda"
 dtype = torch.bfloat16
+N_IMG_EMBS = 3
 
-import matplotlib.pyplot as plt
-import matplotlib
 import logging
-
-
 import os
 import imageio
 import gradio as gr
@@ -24,8 +21,6 @@ from sklearn.svm import SVC
 from sklearn import preprocessing
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-import sched
-import threading
 
 import random
 import time
@@ -104,7 +99,7 @@ pipe = AnimateDiffPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", mot
                                             unet=unet, text_encoder=text_encoder)
 pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear")
 pipe.load_lora_weights("wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora",)
-pipe.set_adapters(["lcm-lora"], [.9])
+pipe.set_adapters(["lcm-lora"], [.95])
 pipe.fuse_lora()
 
 
@@ -121,6 +116,7 @@ pipe.unet.fuse_qkv_projections()
 #pipe.enable_free_init(method="gaussian", use_fast_sampling=True)
 
 pipe.to(device=DEVICE)
+
 #pipe.unet = torch.compile(pipe.unet)
 #pipe.vae = torch.compile(pipe.vae)
 
@@ -130,9 +126,10 @@ pipe.to(device=DEVICE)
 from transformers import AutoProcessor, PaliGemmaForConditionalGeneration, BitsAndBytesConfig
 
 quantization_config = BitsAndBytesConfig(load_in_4bit=True)
-pali = PaliGemmaForConditionalGeneration.from_pretrained('google/paligemma-3b-pt-224', torch_dtype=dtype, device_map='cuda').eval()
+pali = PaliGemmaForConditionalGeneration.from_pretrained('google/paligemma-3b-pt-224', torch_dtype=dtype, quantization_config=quantization_config).eval()
 processor = AutoProcessor.from_pretrained('google/paligemma-3b-pt-224')
 
+#pali = torch.compile(pali)
 
 @spaces.GPU()
 def to_wanted_embs(image_outputs, input_ids, attention_mask, cache_position=None):
@@ -148,19 +145,34 @@ def to_wanted_embs(image_outputs, input_ids, attention_mask, cache_position=None
     return inputs_embeds
     
 
+# TODO cache descriptions?
 @spaces.GPU()
-def generate_pali(user_emb):
-    with torch.no_grad():
-        prompt = 'caption en'
-        model_inputs = processor(text=prompt, images=torch.zeros(1, 3, 224, 224), return_tensors="pt")
-        # we need to get im_embs taken in here.
-        input_len = model_inputs["input_ids"].shape[-1]
-        input_embeds = to_wanted_embs(user_emb.squeeze()[None, None, :].repeat(1, 256, 1), 
-                            model_inputs["input_ids"].to(device), 
-                            model_inputs["attention_mask"].to(device))
+def generate_pali(n_embs):
+    prompt = 'caption en'
+    model_inputs = processor(text=prompt, images=torch.zeros(1, 3, 224, 224), return_tensors="pt")
+    # we need to get im_embs taken in here.
+    
+    descs = ''
+    for n, emb in enumerate(n_embs):
+        if n < len(n_embs)-1:
+            input_len = model_inputs["input_ids"].shape[-1]
+            input_embeds = to_wanted_embs(emb, 
+                                model_inputs["input_ids"].to(device), 
+                                model_inputs["attention_mask"].to(device))
+            generation = pali.generate(max_new_tokens=20, do_sample=True, top_p=.94, temperature=1.2, inputs_embeds=input_embeds)
+            decoded = processor.decode(generation[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)
+            descs += f'Description: {decoded}\n'
+        else:
+            prompt = f'en {descs} Describe a new image that is similar.'
+            print(prompt)
+            model_inputs = processor(text=prompt, images=torch.zeros(1, 3, 224, 224), return_tensors="pt")
+            input_len = model_inputs["input_ids"].shape[-1]
+            input_embeds = to_wanted_embs(emb, 
+                                model_inputs["input_ids"].to(device), 
+                                model_inputs["attention_mask"].to(device))
+            generation = pali.generate(max_new_tokens=20, do_sample=True, top_p=.94, temperature=1.2, inputs_embeds=input_embeds)
+            decoded = processor.decode(generation[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)
     
-        generation = pali.generate(max_new_tokens=100, do_sample=True, top_p=.94, temperature=1.2, inputs_embeds=input_embeds)
-        decoded = processor.decode(generation[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)
     return decoded
 
 
@@ -182,7 +194,7 @@ def generate_gpu(in_im_embs, prompt='the scene'):
         im = torchvision.transforms.ToTensor()(output.frames[0][len(output.frames[0])//2]).unsqueeze(0)
         im = torch.nn.functional.interpolate(im, (224, 224))
         im = (im - .5) * 2
-        gemb = pali.vision_tower(im.to(device).to(dtype)).last_hidden_state.detach().to('cpu').to(torch.float32).mean(1)
+        gemb = pali.vision_tower(im.to(device).to(dtype)).last_hidden_state.detach().to('cpu').to(torch.float32)
     return output, im_emb, gemb
 
 
@@ -210,10 +222,10 @@ def generate(in_im_embs, prompt='the scene'):
 def get_user_emb(embs, ys):
     # handle case where every instance of calibration videos is 'Neither' or 'Like' or 'Dislike'
     
-    if len(list(ys)) <= 7:
-        aways = [.01*torch.randn_like(embs[0]) for i in range(3)]
+    if len(list(ys)) <= 10:
+        aways = [torch.zeros_like(embs[0]) for i in range(10)]
         embs += aways
-        awal = [0 for i in range(3)]
+        awal = [0 for i in range(5)] + [1 for i in range(5)]
         ys += awal
     
     indices = list(range(len(embs)))
@@ -241,9 +253,10 @@ def get_user_emb(embs, ys):
         feature_embs = feature_embs / feature_embs.norm()
     
     #lin_class = Ridge(fit_intercept=False).fit(feature_embs, chosen_y)
-    lin_class = SVC(max_iter=20, kernel='linear', C=.1, class_weight='balanced').fit(feature_embs.squeeze(), chosen_y)
+    #class_weight='balanced'
+    lin_class = SVC(max_iter=500, kernel='linear', C=.1, ).fit(feature_embs.squeeze(), chosen_y)
     coef_ = torch.tensor(lin_class.coef_, dtype=torch.float32).detach().to('cpu')
-    coef_ = coef_ / coef_.abs().max() * 3
+    coef_ = coef_ / coef_.abs().max()
 
     w = 1# if len(embs) % 2 == 0 else 0
     im_emb = w * coef_.to(dtype=dtype)
@@ -273,7 +286,7 @@ def background_next_image():
         # only let it get N (maybe 3) ahead of the user
         #not_rated_rows = prevs_df[[i[1]['user:rating'] == {' ': ' '} for i in prevs_df.iterrows()]]
         rated_rows = prevs_df[[i[1]['user:rating'] != {' ': ' '} for i in prevs_df.iterrows()]]
-        while len(rated_rows) < 4:
+        while len(rated_rows) < 5:
         #    not_rated_rows = prevs_df[[i[1]['user:rating'] == {' ': ' '} for i in prevs_df.iterrows()]]
             rated_rows = prevs_df[[i[1]['user:rating'] != {' ': ' '} for i in prevs_df.iterrows()]]
             time.sleep(.01)
@@ -290,25 +303,21 @@ def background_next_image():
             rated_from_user = rated_rows[[i[1]['from_user_id'] == uid for i in rated_rows.iterrows()]]
 
             # we pop previous ratings if there are > n
-            if len(rated_from_user) >= 15:
+            if len(rated_from_user) >= 25:
                 oldest = rated_from_user.iloc[0]['paths']
                 prevs_df = prevs_df[prevs_df['paths'] != oldest]
             # we don't compute more after n are in the queue for them
-            if len(unrated_from_user) >= 10:
-                continue
-            
-            if len(rated_rows) < 5:
+            if len(unrated_from_user) >= 20:
                 continue
             
             embs, ys, gembs = pluck_embs_ys(uid)
-            
-            user_emb = get_user_emb(embs, ys)
-                        
-            if len(gembs) > 4:
-                user_gem = get_user_emb(gembs, ys) / 4 # TODO scale this correctly; matplotlib, etc.
-                text = generate_pali(user_gem)
+            user_emb = get_user_emb(embs, ys) * 3
+            pos_gembs = [g for g, y in zip(gembs, ys) if y == 1]        
+            if len(pos_gembs) > 4:
+                hist_gem = random.sample(pos_gembs, N_IMG_EMBS) # rng n embeddings
+                text = generate_pali(hist_gem)
             else:
-                text = generate_pali(torch.zeros(1, 1152))
+                text = 'the scene'
             img, embs, new_gem = generate(user_emb, text)
             
             if img:
@@ -351,60 +360,16 @@ def next_image(calibrate_prompts, user_id):
         if len(calibrate_prompts) > 0:
             cal_video = calibrate_prompts.pop(0)
             image = prevs_df[prevs_df['paths'] == cal_video]['paths'].to_list()[0]
-            
             return image, calibrate_prompts, ''
         else:
             embs, ys, gembs = pluck_embs_ys(user_id)
-            user_emb = get_user_emb(embs, ys)
+            user_emb = get_user_emb(embs, ys) * 3
             image, text = pluck_img(user_id, user_emb)
             return image, calibrate_prompts, text
 
 
 
-
-
-
-done_init = False
-
 def start(_, calibrate_prompts, user_id, request: gr.Request):
-    global done_init
-    global prevs_df
-
-    if not done_init:
-        # prep our calibration videos
-        for im in [
-            './first.mp4',
-            # './second.mp4',
-            # './third.mp4',
-            # './fourth.mp4',
-            # './fifth.mp4',
-            # './sixth.mp4',
-            # './seventh.mp4',
-            # './eigth.mp4',
-            # './ninth.mp4',
-            # './tenth.mp4',
-            ]:
-            tmp_df = pd.DataFrame(columns=['paths', 'embeddings', 'ips', 'user:rating', 'text', 'gemb'])
-            tmp_df['paths'] = [im]
-            image = list(imageio.imiter(im))
-            image = image[len(image)//2]
-
-            im = torchvision.transforms.ToTensor()(image).unsqueeze(0)
-            im = torch.nn.functional.interpolate(im, (224, 224))
-            im = (im - .5) * 2
-        
-            im_emb, gemb = encode_space(image, im)
-            im_emb = im_emb.to('cpu')
-            gemb = gemb.to('cpu')
-
-            tmp_df['embeddings'] = [im_emb]
-            tmp_df['gemb'] = [gemb]
-            tmp_df['user:rating'] = [{' ': ' '}]
-            prevs_df = pd.concat((prevs_df, tmp_df))
-        done_init = True
-
-
-
     user_id = int(str(time.time())[-7:].replace('.', ''))
     image, calibrate_prompts, text = next_image(calibrate_prompts, user_id)
     return [
@@ -436,6 +401,7 @@ def choose(img, choice, calibrate_prompts, user_id, request: gr.Request):
         print('NSFW -- choice is disliked')
         choice = 0
     
+    print(prevs_df['paths'].to_list(), img)
     row_mask = [p.split('/')[-1] in img for p in prevs_df['paths'].to_list()]
     # if it's still in the dataframe, add the choice
     if len(prevs_df.loc[row_mask, 'user:rating']) > 0:
@@ -506,11 +472,11 @@ Explore the latent space without text prompts based on your preferences. Learn m
     # calibration videos -- this is a misnomer now :D
     calibrate_prompts = gr.State([
     './first.mp4',
-    # './second.mp4',
-    # './third.mp4',
-    # './fourth.mp4',
-    # './fifth.mp4',
-    # './sixth.mp4',
+    './second.mp4',
+    './third.mp4',
+    './fourth.mp4',
+    './fifth.mp4',
+    './sixth.mp4',
     ])
     def l():
         return None
@@ -569,26 +535,30 @@ scheduler = BackgroundScheduler()
 scheduler.add_job(func=background_next_image, trigger="interval", seconds=.5)
 scheduler.start()
 
-#thread = threading.Thread(target=background_next_image,)
-#thread.start()
 
-# TODO shouldn't call this before gradio launch, yeah?
-@spaces.GPU(duration=50)
-def encode_space(x, im):
-    with torch.no_grad():
-        print('encode')
-        im_emb, _ = pipe.encode_image(
-                    x, DEVICE, 1, output_hidden_state
-                )
-    
-        print('encoded')
-        
-        print('pali_enc')
-        gemb = pali.vision_tower(im.to(dtype).to('cuda')).last_hidden_state
-        
-        print('pali_enced')
-        return im_emb.to('cpu'), gemb.to('cpu')
-
-demo.launch(share=True,)
+# prep our calibration videos
+for im in [
+    './first.mp4',
+    './second.mp4',
+    './third.mp4',
+    './fourth.mp4',
+    './fifth.mp4',
+    './sixth.mp4',
+    './seventh.mp4',
+    './eigth.mp4',
+    './ninth.mp4',
+    './tenth.mp4',
+    ]:
+    tmp_df = pd.DataFrame(columns=['paths', 'embeddings', 'ips', 'user:rating', 'text', 'gemb'])
+    tmp_df['paths'] = [im]
+    image = list(imageio.imiter(im))
+    image = image[len(image)//2]
+    tmp_df['embeddings'] = [torch.load(im.replace('mp4', 'im_.pt'))]
+    tmp_df['gemb'] = [torch.load(im.replace('mp4', 'gemb_.pt'))]
+    tmp_df['user:rating'] = [{' ': ' '}]
+    prevs_df = pd.concat((prevs_df, tmp_df))
+
+
+demo.launch(share=True, server_port=8443)
 
 

lightning_app.py

@@ -1,452 +0,0 @@
-
-import torch
-
-# lol
-sidel = 512
-DEVICE = 'cuda'
-STEPS = 4
-output_hidden_state = False
-device = "cuda"
-dtype = torch.float16
-
-import matplotlib.pyplot as plt
-import matplotlib
-matplotlib.use('TkAgg')
-
-from sklearn.linear_model import LinearRegression
-from sfast.compilers.diffusion_pipeline_compiler import (compile, compile_unet,
-                                                         CompilationConfig)
-config = CompilationConfig.Default()
-
-try:
-    import triton
-    config.enable_triton = True
-except ImportError:
-    print('Triton not installed, skip')
-config.enable_cuda_graph = True
-
-config.enable_jit = True
-config.enable_jit_freeze = True
-
-config.enable_cnn_optimization = True
-config.preserve_parameters = False
-config.prefer_lowp_gemm = True
-
-import imageio
-import gradio as gr
-import numpy as np
-from sklearn.svm import SVC
-from sklearn.inspection import permutation_importance
-from sklearn import preprocessing
-import pandas as pd
-
-import random
-import time
-from PIL import Image
-from safety_checker_improved import maybe_nsfw
-
-
-torch.set_grad_enabled(False)
-torch.backends.cuda.matmul.allow_tf32 = True
-torch.backends.cudnn.allow_tf32 = True
-
-# TODO put back?
-# import spaces
-
-prompt_list = [p for p in list(set(
-                pd.read_csv('./twitter_prompts.csv').iloc[:, 1].tolist())) if type(p) == str]
-
-start_time = time.time()
-
-####################### Setup Model
-from diffusers import AnimateDiffPipeline, MotionAdapter, EulerDiscreteScheduler, LCMScheduler, ConsistencyDecoderVAE, AutoencoderTiny
-from hyper_tile import split_attention, flush
-from huggingface_hub import hf_hub_download
-from safetensors.torch import load_file
-from PIL import Image
-from transformers import CLIPVisionModelWithProjection
-import uuid
-import av
-
-def write_video(file_name, images, fps=10):
-    print('Saving')
-    container = av.open(file_name, mode="w")
-
-    stream = container.add_stream("h264", rate=fps)
-    stream.width = sidel
-    stream.height = sidel
-    stream.pix_fmt = "yuv420p"
-
-    for img in images:
-        img = np.array(img)
-        img = np.round(img).astype(np.uint8)
-        frame = av.VideoFrame.from_ndarray(img, format="rgb24")
-        for packet in stream.encode(frame):
-            container.mux(packet)
-    # Flush stream
-    for packet in stream.encode():
-        container.mux(packet)
-    # Close the file
-    container.close()
-    print('Saved')
-
-bases = {
-    #"basem": "emilianJR/epiCRealism"
-    #SG161222/Realistic_Vision_V6.0_B1_noVAE
-    #runwayml/stable-diffusion-v1-5
-    #frankjoshua/realisticVisionV51_v51VAE
-    #Lykon/dreamshaper-7
-}
-
-image_encoder = CLIPVisionModelWithProjection.from_pretrained("h94/IP-Adapter", subfolder="models/image_encoder", torch_dtype=dtype).to(DEVICE)
-vae = AutoencoderTiny.from_pretrained("madebyollin/taesd", torch_dtype=dtype)
-
-# vae = ConsistencyDecoderVAE.from_pretrained("openai/consistency-decoder", torch_dtype=dtype)
-# vae = compile_unet(vae, config=config)
-
-#adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM")
-#pipe = AnimateDiffPipeline.from_pretrained("emilianJR/epiCRealism", motion_adapter=adapter, image_encoder=image_encoder, torch_dtype=dtype)
-#pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear")
-#pipe.load_lora_weights("wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora",)
-#pipe.set_adapters(["lcm-lora"], [1])
-#pipe.fuse_lora()
-
-pipe = AnimateDiffPipeline.from_pretrained('emilianJR/epiCRealism', torch_dtype=dtype, image_encoder=image_encoder, vae=vae)
-pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing", beta_schedule="linear")
-repo = "ByteDance/AnimateDiff-Lightning"
-ckpt = f"animatediff_lightning_4step_diffusers.safetensors"
-pipe.unet.load_state_dict(load_file(hf_hub_download(repo, ckpt), device='cpu'), strict=False)
-
-
-pipe.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin", map_location='cpu')
-pipe.set_ip_adapter_scale(.8)
-# pipe.unet.fuse_qkv_projections()
-#pipe.enable_free_init(method="gaussian", use_fast_sampling=True)
-
-pipe = compile(pipe, config=config)
-pipe.to(device=DEVICE)
-
-
-# THIS WOULD NEED PATCHING TODO
-with split_attention(pipe.vae, tile_size=128, swap_size=2, disable=False, aspect_ratio=1):
-    # ! Change the tile_size and disable to see their effects
-    with split_attention(pipe.unet, tile_size=128, swap_size=2, disable=False, aspect_ratio=1):
-        im_embs = torch.zeros(1, 1, 1, 1024, device=DEVICE, dtype=dtype)
-        output = pipe(prompt='a person', guidance_scale=0, added_cond_kwargs={}, ip_adapter_image_embeds=[im_embs], num_inference_steps=STEPS)
-        leave_im_emb, _ = pipe.encode_image(
-                output.frames[0][len(output.frames[0])//2], DEVICE, 1, output_hidden_state
-    )
-assert len(output.frames[0]) == 16
-leave_im_emb.to('cpu')
-
-
-# TODO put back
-# @spaces.GPU()
-def generate(prompt, in_im_embs=None, base='basem'):
-
-    if in_im_embs == None:
-        in_im_embs = torch.zeros(1, 1, 1, 1024, device=DEVICE, dtype=dtype)
-        #in_im_embs = in_im_embs / torch.norm(in_im_embs)
-    else:
-        in_im_embs = in_im_embs.to('cuda').unsqueeze(0).unsqueeze(0)
-        #im_embs = torch.cat((torch.zeros(1, 1024, device=DEVICE, dtype=dtype), in_im_embs), 0)
-
-    with split_attention(pipe.unet, tile_size=128, swap_size=2, disable=False, aspect_ratio=1):
-        # ! Change the tile_size and disable to see their effects
-        with split_attention(pipe.vae, tile_size=128, disable=False, aspect_ratio=1):
-            output = pipe(prompt=prompt, guidance_scale=0, added_cond_kwargs={}, ip_adapter_image_embeds=[in_im_embs], num_inference_steps=STEPS)
-
-    im_emb, _ = pipe.encode_image(
-                output.frames[0][len(output.frames[0])//2], DEVICE, 1, output_hidden_state
-            )
-
-    nsfw = maybe_nsfw(output.frames[0][len(output.frames[0])//2])
-    
-    name = str(uuid.uuid4()).replace("-", "")
-    path = f"/tmp/{name}.mp4"
-    
-    if nsfw:
-        gr.Warning("NSFW content detected.")
-        # TODO could return an automatic dislike of auto dislike on the backend for neither as well; just would need refactoring.
-        return None, im_emb
-    
-    plt.close('all')
-    plt.hist(np.array(im_emb.to('cpu')).flatten(), bins=5)
-    plt.savefig('real_im_emb_plot.jpg')
-    
-    write_video(path, output.frames[0])
-    return path, im_emb.to('cpu')
-
-
-#######################
-
-# TODO add to state instead of shared across all
-glob_idx = 0
-
-def next_image(embs, ys, calibrate_prompts):
-    global glob_idx
-    glob_idx = glob_idx + 1
-        
-    with torch.no_grad():
-        if len(calibrate_prompts) > 0:
-            print('######### Calibrating with sample prompts #########')
-            prompt = calibrate_prompts.pop(0)
-            print(prompt)
-            image, img_embs = generate(prompt)
-            embs += img_embs
-            print(len(embs))
-            return image, embs, ys, calibrate_prompts
-        else:
-            print('######### Roaming #########')
-            
-            # sample a .8 of rated embeddings for some stochasticity, or at least two embeddings.
-            # could take a sample < len(embs)
-            #n_to_choose = max(int((len(embs))), 2)
-            #indices = random.sample(range(len(embs)), n_to_choose)
-            
-            # sample only as many negatives as there are positives
-            #pos_indices = [i for i in indices if ys[i] == 1]
-            #neg_indices = [i for i in indices if ys[i] == 0]
-            #lower = min(len(pos_indices), len(neg_indices))
-            #neg_indices = random.sample(neg_indices, lower)
-            #pos_indices = random.sample(pos_indices, lower)
-            #indices = neg_indices + pos_indices
-            
-            pos_indices = [i for i in range(len(embs)) if ys[i] == 1]
-            neg_indices = [i for i in range(len(embs)) if ys[i] == 0]
-            
-            # the embs & ys stay tied by index but we shuffle to drop randomly
-            random.shuffle(pos_indices)
-            random.shuffle(neg_indices)
-            
-            #if len(pos_indices) - len(neg_indices) > 48 and len(pos_indices) > 80:
-            #    pos_indices = pos_indices[32:]
-            if len(neg_indices) - len(pos_indices) > 48/16 and len(pos_indices) > 120/16:
-                pos_indices = pos_indices[1:]
-            if len(neg_indices) - len(pos_indices) > 48/16 and len(neg_indices) > 200/16:
-                neg_indices = neg_indices[2:]
-            
-            
-            print(len(pos_indices), len(neg_indices))
-            indices = pos_indices + neg_indices
-            
-            embs = [embs[i] for i in indices]
-            ys = [ys[i] for i in indices]
-            indices = list(range(len(embs)))
-            
-            
-            # handle case where every instance of calibration prompts is 'Neither' or 'Like' or 'Dislike'
-            if len(list(set(ys))) <= 1:
-                embs.append(.01*torch.randn(1024))
-                embs.append(.01*torch.randn(1024))
-                ys.append(0)
-                ys.append(1)
-
-            
-            # also add the latest 0 and the latest 1
-            has_0 = False
-            has_1 = False
-            for i in reversed(range(len(ys))):
-                if ys[i] == 0 and has_0 == False:
-                    indices.append(i)
-                    has_0 = True
-                elif ys[i] == 1 and has_1 == False:
-                    indices.append(i)
-                    has_1 = True
-                if has_0 and has_1:
-                    break
-                    
-            # we may have just encountered a rare multi-threading diffusers issue (https://github.com/huggingface/diffusers/issues/5749);
-            # this ends up adding a rating but losing an embedding, it seems.
-            # let's take off a rating if so to continue without indexing errors.
-            if len(ys) > len(embs):
-                print('ys are longer than embs; popping latest rating')
-                ys.pop(-1)
-            
-            feature_embs = np.array(torch.stack([embs[i].to('cpu') for i in indices] + [leave_im_emb[0].to('cpu')]).to('cpu'))
-            scaler = preprocessing.StandardScaler().fit(feature_embs)
-            feature_embs = scaler.transform(feature_embs)
-            chosen_y = np.array([ys[i] for i in indices] + [0])
-            
-            print('Gathering coefficients')
-            #lin_class = LinearRegression(fit_intercept=False).fit(feature_embs, chosen_y)
-            lin_class = SVC(max_iter=50000, kernel='linear', class_weight='balanced', C=1).fit(feature_embs, chosen_y)
-            coef_ = torch.tensor(lin_class.coef_, dtype=torch.double)
-            coef_ = coef_ / coef_.abs().max() * 3
-            print(coef_.shape, 'COEF')
-
-            plt.close('all')
-            plt.hist(np.array(coef_).flatten(), bins=5)
-            plt.savefig('plot.jpg')
-            print(coef_)
-            print('Gathered')
-
-            rng_prompt = random.choice(prompt_list)
-            w = 1# if len(embs) % 2 == 0 else 0
-            im_emb = w * coef_.to(dtype=dtype)
-
-            prompt= 'the scene' if glob_idx % 2 == 0 else rng_prompt
-            print(prompt)
-            image, im_emb = generate(prompt, im_emb)
-            embs += im_emb
-            
-            if len(embs) > 700/16:
-                embs = embs[1:]
-                ys = ys[1:]
-            
-            return image, embs, ys, calibrate_prompts
-
-
-
-
-
-
-
-
-
-def start(_, embs, ys, calibrate_prompts):
-    image, embs, ys, calibrate_prompts = next_image(embs, ys, calibrate_prompts)
-    return [
-            gr.Button(value='Like (L)', interactive=True), 
-            gr.Button(value='Neither (Space)', interactive=True), 
-            gr.Button(value='Dislike (A)', interactive=True),
-            gr.Button(value='Start', interactive=False),
-            image,
-            embs,
-            ys,
-            calibrate_prompts
-            ]
-
-
-def choose(img, choice, embs, ys, calibrate_prompts):
-    if choice == 'Like (L)':
-        choice = 1
-    elif choice == 'Neither (Space)':
-        embs = embs[:-1]
-        img, embs, ys, calibrate_prompts = next_image(embs, ys, calibrate_prompts)
-        return img, embs, ys, calibrate_prompts
-    else:
-        choice = 0
-
-    # if we detected NSFW, leave that area of latent space regardless of how they rated chosen.
-    # TODO skip allowing rating
-    if img == None:
-        print('NSFW -- choice is disliked')
-        choice = 0
-    
-    ys += [choice]*1
-    img, embs, ys, calibrate_prompts = next_image(embs, ys, calibrate_prompts)
-    return img, embs, ys, calibrate_prompts
-
-css = '''.gradio-container{max-width: 700px !important}
-#description{text-align: center}
-#description h1, #description h3{display: block}
-#description p{margin-top: 0}
-.fade-in-out {animation: fadeInOut 3s forwards}
-@keyframes fadeInOut {
-    0% {
-      background: var(--bg-color);
-    }
-    100% {
-      background: var(--button-secondary-background-fill);
-    }
-}
-'''
-js_head = '''
-<script>
-document.addEventListener('keydown', function(event) {
-    if (event.key === 'a' || event.key === 'A') {
-        // Trigger click on 'dislike' if 'A' is pressed
-        document.getElementById('dislike').click();
-    } else if (event.key === ' ' || event.keyCode === 32) {
-        // Trigger click on 'neither' if Spacebar is pressed
-        document.getElementById('neither').click();
-    } else if (event.key === 'l' || event.key === 'L') {
-        // Trigger click on 'like' if 'L' is pressed
-        document.getElementById('like').click();
-    }
-});
-function fadeInOut(button, color) {
-  button.style.setProperty('--bg-color', color);
-  button.classList.remove('fade-in-out');
-  void button.offsetWidth; // This line forces a repaint by accessing a DOM property
-  
-  button.classList.add('fade-in-out');
-  button.addEventListener('animationend', () => {
-    button.classList.remove('fade-in-out'); // Reset the animation state
-  }, {once: true});
-}
-document.body.addEventListener('click', function(event) {
-    const target = event.target;
-    if (target.id === 'dislike') {
-      fadeInOut(target, '#ff1717');
-    } else if (target.id === 'like') {
-      fadeInOut(target, '#006500');
-    } else if (target.id === 'neither') {
-      fadeInOut(target, '#cccccc');
-    }
-});
-
-</script>
-'''
-
-with gr.Blocks(css=css, head=js_head) as demo:
-    gr.Markdown('''### Blue Tigers: Generative Recommenders for Exporation of Video
-    Explore the latent space without text prompts based on your preferences. Learn more on [the write-up](https://rynmurdock.github.io/posts/2024/3/generative_recomenders/).
-    ''', elem_id="description")
-    embs = gr.State([])
-    ys = gr.State([])
-    calibrate_prompts = gr.State([
-    'the moon is melting into my glass of tea',
-    'a sea slug -- pair of claws scuttling -- jelly fish glowing',
-    'an adorable creature. It may be a goblin or a pig or a slug.',
-    'an animation about a gorgeous nebula',
-    'an octopus writhes',
-    ])
-    def l():
-        return None
-
-    with gr.Row(elem_id='output-image'):
-        img = gr.Video(
-        label='Lightning',
-        autoplay=True,
-        interactive=False,
-        height=sidel,
-        width=sidel,
-        include_audio=False,
-        elem_id="video_output"
-       )
-        img.play(l, js='''document.querySelector('[data-testid="Lightning-player"]').loop = true''')
-    with gr.Row(equal_height=True):
-        b3 = gr.Button(value='Dislike (A)', interactive=False, elem_id="dislike")
-        b2 = gr.Button(value='Neither (Space)', interactive=False, elem_id="neither")
-        b1 = gr.Button(value='Like (L)', interactive=False, elem_id="like")
-        b1.click(
-        choose, 
-        [img, b1, embs, ys, calibrate_prompts],
-        [img, embs, ys, calibrate_prompts]
-        )
-        b2.click(
-        choose, 
-        [img, b2, embs, ys, calibrate_prompts],
-        [img, embs, ys, calibrate_prompts]
-        )
-        b3.click(
-        choose, 
-        [img, b3, embs, ys, calibrate_prompts],
-        [img, embs, ys, calibrate_prompts]
-        )
-    with gr.Row():
-        b4 = gr.Button(value='Start')
-        b4.click(start,
-                 [b4, embs, ys, calibrate_prompts],
-                 [b1, b2, b3, b4, img, embs, ys, calibrate_prompts])
-    with gr.Row():
-        html = gr.HTML('''<div style='text-align:center; font-size:20px'>You will calibrate for several prompts and then roam. </ div><br><br><br>
-<div style='text-align:center; font-size:14px'>Note that while the AnimateDiff-Lightning model with NSFW filtering is unlikely to produce NSFW images, this may still occur, and users should avoid NSFW content when rating.
-</ div>
-<br><br>
-<div style='text-align:center; font-size:14px'>Thanks to @multimodalart for their contributions to the demo, esp. the interface and @maxbittker for feedback.
-</ div>''')
-
-demo.launch(share=True)

requirements.txt

@@ -15,6 +15,4 @@ tensorflow==2.14.0
 imageio
 apscheduler
 pandas
-av
-torchvision
-bitsandbytes
+av

twitter_prompts.csv

@@ -1,72 +0,0 @@
-,0
-0,a sunset
-1,a still life in blue
-2,last day on earth
-3,the conch shell
-4,the winds of change
-5,a surrealist eye
-6,a surrealist polaroid photo of an apple
-7,metaphysics
-8,the sun is setting into my glass of tea
-9,the moon at 3am
-10,a memento mori
-11,quaking aspen tree
-12,violets and daffodils
-13,espresso
-14,sisyphus
-15,high windows of stained glass
-16,a green dog
-17,an adorable companion; it is a pig
-18,bird of paradise
-19,a complex intricate machine
-20,a white clock
-21,a film featuring the landscape Salt Lake City Utah
-22,a creature
-23,a house set aflame.
-24,a gorgeous landscape by Cy Twombly
-25,smoke rises from the caterpillar's hookah
-26,corvid in red
-27,Monet's pond
-28,Genesis
-29,Death is a black camel that kneels down so we can ride
-30,a cherry tree made of fractals
-29,the end of the sidewalk
-30,a polaroid photo of a bustling city of lights and sky scrapers
-31,The Fig Tree metaphor
-32,God killed Van Gogh.
-33,a cosmic entity alien with four eyes.
-34,a horse with 128 eyes.
-35,a being with an infinite set of eyes (it is omniscient)
-36,A sticky-note magnum opus featuring birds
-37,Moka Pot
-38,the moon is a sickle cell
-39,The Penultimate Supper
-40,Art
-41,surrealism
-42,a god made of wires & dust
-43,a dandelion blown into the universe
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-