update

Video-P2P/configs/man-surfing.yaml

@@ -0,0 +1,41 @@
+pretrained_model_path: "./checkpoints/stable-diffusion-v1-4"
+output_dir: "./outputs/man-surfing"
+
+train_data:
+  video_path: "data/man-surfing.mp4"
+  prompt: "a man is surfing"
+  n_sample_frames: 24
+  width: 512
+  height: 512
+  sample_start_idx: 0
+  sample_frame_rate: 1
+
+validation_data:
+  prompts:
+    - "a panda is surfing"
+    - "a boy, wearing a birthday hat, is surfing"
+    - "a raccoon is surfing, cartoon style"
+    - "Iron Man is surfing in the desert"
+  video_length: 24
+  width: 512
+  height: 512
+  num_inference_steps: 50
+  guidance_scale: 12.5
+  use_inv_latent: True
+  num_inv_steps: 50
+
+learning_rate: 3e-5
+train_batch_size: 1
+max_train_steps: 500
+checkpointing_steps: 1000
+validation_steps: 100
+trainable_modules:
+  - "attn1.to_q"
+  - "attn2.to_q"
+  - "attn_temp"
+
+seed: 33
+mixed_precision: fp16
+use_8bit_adam: False
+gradient_checkpointing: True
+enable_xformers_memory_efficient_attention: True

app.py

@@ -8,15 +8,15 @@ from subprocess import getoutput
 import gradio as gr
 import torch
 
-# from app_inference import create_inference_demo
+from app_inference import create_inference_demo
 from app_training import create_training_demo
-# from app_upload import create_upload_demo
+from app_upload import create_upload_demo
 from inference import InferencePipeline
 from trainer import Trainer
 
 TITLE = '# [Video-P2P](https://video-p2p.github.io/) UI'
 
-ORIGINAL_SPACE_ID = 'Shaldon/Video-P2P-Training-UI'
+ORIGINAL_SPACE_ID = 'Shaldon/Video-P2P-Demo'
 SPACE_ID = os.getenv('SPACE_ID', ORIGINAL_SPACE_ID)
 GPU_DATA = getoutput('nvidia-smi')
 SHARED_UI_WARNING = f'''## Attention - Training doesn't work in this shared UI. You can duplicate and use it with a paid private T4 GPU.
@@ -70,13 +70,13 @@ with gr.Blocks(css='style.css') as demo:
     with gr.Tabs():
         with gr.TabItem('Train'):
             create_training_demo(trainer, pipe)
-        # with gr.TabItem('Run'):
-        #     create_inference_demo(pipe, HF_TOKEN)
-        # with gr.TabItem('Upload'):
-        #     gr.Markdown('''
-        #     - You can use this tab to upload models later if you choose not to upload models in training time or if upload in training time failed.
-        #     ''')
-        #     create_upload_demo(HF_TOKEN)
+        with gr.TabItem('Run'):
+            create_inference_demo(pipe, HF_TOKEN)
+        with gr.TabItem('Upload'):
+            gr.Markdown('''
+            - You can use this tab to upload models later if you choose not to upload models in training time or if upload in training time failed.
+            ''')
+            create_upload_demo(HF_TOKEN)
 
     if not HF_TOKEN:
         show_warning(HF_TOKEN_NOT_SPECIFIED_WARNING)

inference.py

@@ -13,7 +13,7 @@ from diffusers.utils.import_utils import is_xformers_available
 from einops import rearrange
 from huggingface_hub import ModelCard
 
-sys.path.append('Video-P2P-Demo')
+sys.path.append('Video-P2P')
 
 from tuneavideo.models.unet import UNet3DConditionModel
 from tuneavideo.pipelines.pipeline_tuneavideo import TuneAVideoPipeline

trainer.py

@@ -17,10 +17,10 @@ from omegaconf import OmegaConf
 from app_upload import ModelUploader
 from utils import save_model_card
 
-sys.path.append('Video-P2P-Demo')
+sys.path.append('Video-P2P')
 
 # URL_TO_JOIN_MODEL_LIBRARY_ORG = 'https://huggingface.co/organizations/Tune-A-Video-library/share/YjTcaNJmKyeHFpMBioHhzBcTzCYddVErEk'
-ORIGINAL_SPACE_ID = 'Shaldon/Video-P2P-Training-UI'
+ORIGINAL_SPACE_ID = 'Shaldon/Video-P2P-Demo'
 SPACE_ID = os.getenv('SPACE_ID', ORIGINAL_SPACE_ID)
 
 
@@ -103,7 +103,7 @@ class Trainer:
         #     self.join_model_library_org(
         #         self.hf_token if self.hf_token else input_token)
 
-        config = OmegaConf.load('Video-P2P-Demo/configs/rabbit-jump-tune.yaml')
+        config = OmegaConf.load('Video-P2P/configs/man-surfing.yaml')
         config.pretrained_model_path = self.download_base_model(base_model)
         config.output_dir = output_dir.as_posix()
         config.train_data.video_path = training_video.name  # type: ignore
@@ -133,7 +133,7 @@ class Trainer:
         with open(config_path, 'w') as f:
             OmegaConf.save(config, f)
 
-        command = f'accelerate launch Video-P2P-Demo/run_tuning.py --config {config_path}'
+        command = f'accelerate launch Video-P2P/run_tuning.py --config {config_path}'
         subprocess.run(shlex.split(command))
         save_model_card(save_dir=output_dir,
                         base_model=base_model,