feat(wip): lycoris

app.py

@@ -2,10 +2,13 @@ import torch
 import gradio as gr
 import spaces
 import random
+import os
 from diffusers.utils import export_to_video
 from diffusers import AutoencoderKLWan, WanPipeline
 from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
 from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
+from huggingface_hub import hf_hub_download
+from lycoris import create_lycoris_from_weights
 
 # Define model options
 MODEL_OPTIONS = {
@@ -19,14 +22,39 @@ SCHEDULER_OPTIONS = {
     "FlowMatchEulerDiscreteScheduler": FlowMatchEulerDiscreteScheduler
 }
 
+def download_adapter(repo_id, weight_name=None):
+    """
+    Download the adapter file from the Hugging Face Hub.
+    If weight_name is not provided, attempts to use pytorch_lora_weights.safetensors
+    """
+    adapter_filename = weight_name if weight_name else "pytorch_lora_weights.safetensors"
+    cache_dir = os.environ.get('HF_PATH', os.path.expanduser('~/.cache/huggingface/hub/models'))
+    cleaned_adapter_path = repo_id.replace("/", "_").replace("\\", "_").replace(":", "_")
+    path_to_adapter = os.path.join(cache_dir, cleaned_adapter_path)
+    os.makedirs(path_to_adapter, exist_ok=True)
+    
+    try:
+        path_to_adapter_file = hf_hub_download(
+            repo_id=repo_id, 
+            filename=adapter_filename, 
+            local_dir=path_to_adapter
+        )
+        return path_to_adapter_file
+    except Exception as e:
+        # If specific file not found, try to get a list of available safetensors files
+        if weight_name is None:
+            raise ValueError(f"Could not download default adapter file: {str(e)}\nPlease specify the exact weight file name.")
+        else:
+            raise ValueError(f"Could not download adapter file {weight_name}: {str(e)}")
+
 @spaces.GPU(duration=300)
 def generate_video(
     model_choice,
     prompt,
     negative_prompt,
-    lora_id,
-    lora_weight_name,
-    lora_scale,
+    lycoris_id,
+    lycoris_weight_name,
+    lycoris_scale,
     scheduler_type,
     flow_shift,
     height,
@@ -65,24 +93,22 @@ def generate_video(
     # Move to GPU
     pipe.to("cuda")
     
-    # Load LoRA weights if provided
-    if lora_id and lora_id.strip():
+    # Load LyCORIS weights if provided
+    if lycoris_id and lycoris_id.strip():
         try:
-            # If a specific weight name is provided, use it
-            if lora_weight_name and lora_weight_name.strip():
-                pipe.load_lora_weights(lora_id, weight_name=lora_weight_name)
-            else:
-                pipe.load_lora_weights(lora_id)
+            # Download the adapter file
+            adapter_file_path = download_adapter(repo_id=lycoris_id, weight_name=lycoris_weight_name if lycoris_weight_name and lycoris_weight_name.strip() else None)
+            
+            # Apply LyCORIS adapter
+            wrapper, *_ = create_lycoris_from_weights(lycoris_scale, adapter_file_path, pipe.transformer)
+            wrapper.merge_to()
             
-            # Set lora scale if applicable
-            if hasattr(pipe, "set_adapters_scale") and lora_scale is not None:
-                pipe.set_adapters_scale(lora_scale)
         except ValueError as e:
-            # Return informative error if there are multiple safetensors and no weight name
-            if "more than one weights file" in str(e):
-                return f"Error: The repository '{lora_id}' contains multiple safetensors files. Please specify a weight name using the 'LoRA Weight Name' field.", seed
+            # Return informative error if there are issues loading the adapter
+            if "more than one weights file" in str(e) or "Could not download default adapter file" in str(e):
+                return f"Error: The repository '{lycoris_id}' may contain multiple weight files. Please specify a weight name using the 'LyCORIS Weight Name' field.", seed
             else:
-                return f"Error loading LoRA weights: {str(e)}", seed
+                return f"Error loading LyCORIS weights: {str(e)}", seed
     
     # Enable CPU offload for low VRAM
     pipe.enable_model_cpu_offload()
@@ -114,7 +140,7 @@ with gr.Blocks() as demo:
         </svg>
     </p>
     """)
-    gr.Markdown("# Wan 2.1 T2V 1.3B with LoRA")
+    gr.Markdown("# Wan 2.1 T2V with LyCORIS")
     
     with gr.Row():
         with gr.Column(scale=1):
@@ -137,19 +163,19 @@ with gr.Blocks() as demo:
             )
             
             with gr.Row():
-                lora_id = gr.Textbox(
-                    label="LoRA Model Repo (e.g., TheBulge/AndroWan-2.1-T2V-1.3B)",
+                lycoris_id = gr.Textbox(
+                    label="LyCORIS Model Repo (e.g., TheBulge/AndroWan-2.1-T2V-1.3B)",
                     value="TheBulge/AndroWan-2.1-T2V-1.3B"
                 )
             
             with gr.Row():
-                lora_weight_name = gr.Textbox(
-                    label="LoRA Path in Repo (optional)",
+                lycoris_weight_name = gr.Textbox(
+                    label="LyCORIS Weight Path in Repo (optional)",
                     value="safetensors/AndroWan_v32-0036_ema.safetensors",
                     info="Specify for repos with multiple .safetensors files, e.g.: adapter_model.safetensors, pytorch_lora_weights.safetensors, etc."
                 )
-                lora_scale = gr.Slider(
-                    label="LoRA Scale",
+                lycoris_scale = gr.Slider(
+                    label="LyCORIS Scale",
                     minimum=0.0,
                     maximum=2.0,
                     value=1.00,
@@ -238,9 +264,9 @@ with gr.Blocks() as demo:
             model_choice,
             prompt,
             negative_prompt,
-            lora_id,
-            lora_weight_name,
-            lora_scale,
+            lycoris_id,
+            lycoris_weight_name,
+            lycoris_scale,
             scheduler_type,
             flow_shift,
             height,
@@ -261,8 +287,8 @@ with gr.Blocks() as demo:
     - Number of frames should be of the form 4k+1 (e.g., 33, 81)
     - Stick to lower frame counts. Even at 480p, an 81 frame sequence at 30 steps will nearly time out the request in this space.
     
-    ## Using LoRAs with multiple safetensors files:
-    If you encounter an error stating "more than one weights file", you need to specify the exact weight file name in the "LoRA Weight Name" field.
+    ## Using LyCORIS with multiple safetensors files:
+    If you encounter an error stating "Could not download default adapter file", you need to specify the exact weight file name in the "LyCORIS Weight Name" field.
     You can find this by browsing the repository on Hugging Face and looking for the safetensors files (common names include: adapter_model.safetensors, pytorch_lora_weights.safetensors).
     """)
 

requirements.txt

@@ -7,4 +7,6 @@ ftfy>=6.1.3
 einops>=0.7.0
 imageio>=2.31.6
 imageio-ffmpeg>=0.4.9
-peft==0.7.1
+peft==0.7.1
+lycoris-lora>=0.1.5
+huggingface_hub>=0.19.0