| | from utils.distributed import launch_distributed_job |
| | from utils.scheduler import FlowMatchScheduler |
| | from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder |
| | from utils.dataset import TextDataset |
| | import torch.distributed as dist |
| | from tqdm import tqdm |
| | import argparse |
| | import torch |
| | import math |
| | import os |
| |
|
| |
|
| | def init_model(device): |
| | model = WanDiffusionWrapper().to(device).to(torch.float32) |
| | encoder = WanTextEncoder().to(device).to(torch.float32) |
| | model.model.requires_grad_(False) |
| |
|
| | scheduler = FlowMatchScheduler( |
| | shift=8.0, sigma_min=0.0, extra_one_step=True) |
| | scheduler.set_timesteps(num_inference_steps=48, denoising_strength=1.0) |
| | scheduler.sigmas = scheduler.sigmas.to(device) |
| |
|
| | sample_neg_prompt = '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走' |
| |
|
| | unconditional_dict = encoder( |
| | text_prompts=[sample_neg_prompt] |
| | ) |
| |
|
| | return model, encoder, scheduler, unconditional_dict |
| |
|
| |
|
| | def main(): |
| | parser = argparse.ArgumentParser() |
| | parser.add_argument("--local_rank", type=int, default=-1) |
| | parser.add_argument("--output_folder", type=str) |
| | parser.add_argument("--caption_path", type=str) |
| | parser.add_argument("--guidance_scale", type=float, default=6.0) |
| |
|
| | args = parser.parse_args() |
| |
|
| | |
| | launch_distributed_job() |
| |
|
| | device = torch.cuda.current_device() |
| |
|
| | torch.set_grad_enabled(False) |
| | torch.backends.cuda.matmul.allow_tf32 = True |
| | torch.backends.cudnn.allow_tf32 = True |
| |
|
| | model, encoder, scheduler, unconditional_dict = init_model(device=device) |
| |
|
| | dataset = TextDataset(args.caption_path) |
| |
|
| | |
| | os.makedirs(args.output_folder, exist_ok=True) |
| |
|
| | for index in tqdm(range(int(math.ceil(len(dataset) / dist.get_world_size()))), disable=dist.get_rank() != 0): |
| | prompt_index = index * dist.get_world_size() + dist.get_rank() |
| | if prompt_index >= len(dataset): |
| | continue |
| | prompt = dataset[prompt_index] |
| |
|
| | conditional_dict = encoder(text_prompts=prompt) |
| |
|
| | latents = torch.randn( |
| | [1, 21, 16, 60, 104], dtype=torch.float32, device=device |
| | ) |
| |
|
| | noisy_input = [] |
| |
|
| | for progress_id, t in enumerate(tqdm(scheduler.timesteps)): |
| | timestep = t * \ |
| | torch.ones([1, 21], device=device, dtype=torch.float32) |
| |
|
| | noisy_input.append(latents) |
| |
|
| | _, x0_pred_cond = model( |
| | latents, conditional_dict, timestep |
| | ) |
| |
|
| | _, x0_pred_uncond = model( |
| | latents, unconditional_dict, timestep |
| | ) |
| |
|
| | x0_pred = x0_pred_uncond + args.guidance_scale * ( |
| | x0_pred_cond - x0_pred_uncond |
| | ) |
| |
|
| | flow_pred = model._convert_x0_to_flow_pred( |
| | scheduler=scheduler, |
| | x0_pred=x0_pred.flatten(0, 1), |
| | xt=latents.flatten(0, 1), |
| | timestep=timestep.flatten(0, 1) |
| | ).unflatten(0, x0_pred.shape[:2]) |
| |
|
| | latents = scheduler.step( |
| | flow_pred.flatten(0, 1), |
| | scheduler.timesteps[progress_id] * torch.ones( |
| | [1, 21], device=device, dtype=torch.long).flatten(0, 1), |
| | latents.flatten(0, 1) |
| | ).unflatten(dim=0, sizes=flow_pred.shape[:2]) |
| |
|
| | noisy_input.append(latents) |
| |
|
| | noisy_inputs = torch.stack(noisy_input, dim=1) |
| |
|
| | noisy_inputs = noisy_inputs[:, [0, 12, 24, 36, -1]] |
| |
|
| | stored_data = noisy_inputs |
| |
|
| | torch.save( |
| | {prompt: stored_data.cpu().detach()}, |
| | os.path.join(args.output_folder, f"{prompt_index:05d}.pt") |
| | ) |
| |
|
| | dist.barrier() |
| |
|
| |
|
| | if __name__ == "__main__": |
| | main() |
| |
|
