| # Model Setting | |
| pretrained_model_name_or_path: stabilityai/stable-video-diffusion-img2vid # stabilityai/pretrained | |
| load_unet_path: ../model_weights/ROB_regular_weights/v4_VL_paper/checkpoint-99000 # None/specific path This is for pretrained-UNet path | |
| load_controlnet_path: # None/specific path For checkpoint loaded from pretrained-Controlnet Path | |
| video_seq_length: 14 | |
| process_fps: 7 | |
| train_noise_aug_strength: 0.1 | |
| scheduler: EDM | |
| conditioning_dropout_prob: 0.1 | |
| # Dataset Setting | |
| data_loader_type: thisthat # thisthat | |
| dataset_name: Bridge # Bridge | |
| dataset_path: [../sanity_check/bridge_v1_TT14, ../sanity_check/bridge_v2_TT14] # ../Bridge_filter_flow, ../Bridge_v2_filter_flow/] | |
| output_dir: checkpoints/img2video | |
| height: 256 # Ratio that is functional: 256:384 576:1024 320:448 320:576 512:640 448:640 | |
| width: 384 # It is said that the height and width should be a scale of 64 | |
| dataloader_num_workers: 4 # For Debug, it only needs 1 | |
| flip_aug_prob: 0.45 # Whether we flip the GT and cond vertically | |
| # No acceleration_tolerance, since TT dataset already filter those out | |
| # Text setting | |
| use_text: True # If this is True, we will use text value | |
| pretrained_tokenizer_name_or_path: stabilityai/stable-diffusion-2-1-base # Use SD 2.1 | |
| empty_prompts_proportion: 0.0 | |
| mix_ambiguous: False # Whether we mix ambiguous prompt for "this" and "that" | |
| # Mask setting | |
| mask_unet_vae: False # Whether we use mask to map latents to be zero padding | |
| mask_controlnet_vae: False | |
| mask_proportion: 0.0 | |
| # Condition Setting | |
| conditioning_channels: 3 # Usually it is 3 | |
| num_points_left: # 1 # For flow: You can only choose one between flow_select_rate and num_points_left; num_points_left should be higher priority | |
| flow_select_rate: 0.99 # For flow | |
| threshold_factor: 0.2 # For flow | |
| dilate: True # Traj must be True for dilate | |
| inner_conditioning_scale: 1.0 # Conditioning scale for the internal value, defauly is starting from 1.0 | |
| outer_conditioning_scale: 1.0 # Outer Conditioning Scale for whole conditioning trainable copy 这里有点意思，直接不小心设定成2.0了 | |
| # Motion setting | |
| motion_bucket_id: 200 | |
| dataset_motion_mean: 25 # For 14 fps, it is N(25, 10) | |
| dataset_motion_std: 10 # For 25 fps, it is N(18, 7) | |
| svd_motion_mean: 180 | |
| svd_motion_std: 30 | |
| # Training setting | |
| resume_from_checkpoint: False # latest/False | |
| num_train_iters: 30100 # Will automatically choose the checkpoints | |
| partial_finetune: False # Whether we just tune some params to speed up | |
| train_batch_size: 1 # This is the batch size per GPU | |
| checkpointing_steps: 3000 | |
| validation_step: 300 | |
| logging_name: logging | |
| seed: 42 | |
| validation_img_folder: datasets/validation_TT14 | |
| validation_store_folder: validation_videos | |
| checkpoints_total_limit: 15 | |
| # Noise Strength | |
| noise_mean: 0.5 # Regular Img2Video: (0.7, 1.6); Text2Video: (0.5, 1.4) | |
| noise_std: 1.4 | |
| # Inference | |
| num_inference_steps: 25 | |
| use_instructpix2pix: False # Whether we will use the instructPix2Pix mode, which involves 3 inputs; it may needs tuning to have better result at the end. | |
| inference_noise_aug_strength: 0.1 | |
| inference_max_guidance_scale: 3.0 # Take training and testing at different scenario | |
| inference_guess_mode: False # Whether we use guess mode in the contorlnet | |
| image_guidance_scale: 2.5 # Empirically, 2.5 is the best value Seems not using this now | |
| # Learning Rate and Optimizer | |
| learning_rate: 5e-6 # 5e-6 is the LR we test that is just right | |
| scale_lr: False # TODO: Is it needed to scale the learning rate? | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.999 | |
| use_8bit_adam: True # Need this to save more memory | |
| adam_weight_decay: 1e-2 | |
| adam_epsilon: 1e-08 | |
| lr_warmup_steps: 500 | |
| lr_decay_scale: 0.5 | |
| # Other Setting | |
| mixed_precision: fp16 | |
| gradient_accumulation_steps: 1 # ???? | |
| gradient_checkpointing: 1 # ???? | |
| report_to: tensorboard |