multimodalart HF staff commited on
Commit
bfdbdf6
1 Parent(s): 9465fd2

add_stufff

Browse files
Files changed (7) hide show
  1. app.py +0 -0
  2. arrow.png +0 -0
  3. cat-toy-deprec.png +0 -0
  4. cat-toy.png +0 -0
  5. duplicate.png +0 -0
  6. mix.zip +3 -0
  7. train_dreambooth.py +787 -0
app.py CHANGED
The diff for this file is too large to render. See raw diff
 
arrow.png ADDED
cat-toy-deprec.png ADDED
cat-toy.png ADDED
duplicate.png ADDED
mix.zip ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e76812bdb3b21d1ef3050c6c6a09b09a16a21ae23476e92cd3dd1dfa2e846b22
3
+ size 61281901
train_dreambooth.py ADDED
@@ -0,0 +1,787 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import itertools
3
+ import math
4
+ import os
5
+ from pathlib import Path
6
+ from typing import Optional
7
+ import subprocess
8
+ import sys
9
+
10
+ import torch
11
+ import torch.nn.functional as F
12
+ import torch.utils.checkpoint
13
+ from torch.utils.data import Dataset
14
+
15
+ from accelerate import Accelerator
16
+ from accelerate.logging import get_logger
17
+ from accelerate.utils import set_seed
18
+ from diffusers import AutoencoderKL, DDPMScheduler, StableDiffusionPipeline, UNet2DConditionModel
19
+ from diffusers.optimization import get_scheduler
20
+ from huggingface_hub import HfFolder, Repository, whoami
21
+ from PIL import Image
22
+ from torchvision import transforms
23
+ from tqdm.auto import tqdm
24
+ from transformers import CLIPTextModel, CLIPTokenizer
25
+
26
+
27
+ logger = get_logger(__name__)
28
+
29
+
30
+ def parse_args():
31
+ parser = argparse.ArgumentParser(description="Simple example of a training script.")
32
+ parser.add_argument(
33
+ "--pretrained_model_name_or_path",
34
+ type=str,
35
+ default=None,
36
+ required=True,
37
+ help="Path to pretrained model or model identifier from huggingface.co/models.",
38
+ )
39
+ parser.add_argument(
40
+ "--tokenizer_name",
41
+ type=str,
42
+ default=None,
43
+ help="Pretrained tokenizer name or path if not the same as model_name",
44
+ )
45
+ parser.add_argument(
46
+ "--instance_data_dir",
47
+ type=str,
48
+ default=None,
49
+ required=True,
50
+ help="A folder containing the training data of instance images.",
51
+ )
52
+ parser.add_argument(
53
+ "--class_data_dir",
54
+ type=str,
55
+ default=None,
56
+ required=False,
57
+ help="A folder containing the training data of class images.",
58
+ )
59
+ parser.add_argument(
60
+ "--instance_prompt",
61
+ type=str,
62
+ default=None,
63
+ help="The prompt with identifier specifying the instance",
64
+ )
65
+ parser.add_argument(
66
+ "--class_prompt",
67
+ type=str,
68
+ default="",
69
+ help="The prompt to specify images in the same class as provided instance images.",
70
+ )
71
+ parser.add_argument(
72
+ "--with_prior_preservation",
73
+ default=False,
74
+ action="store_true",
75
+ help="Flag to add prior preservation loss.",
76
+ )
77
+ parser.add_argument("--prior_loss_weight", type=float, default=1.0, help="The weight of prior preservation loss.")
78
+ parser.add_argument(
79
+ "--num_class_images",
80
+ type=int,
81
+ default=100,
82
+ help=(
83
+ "Minimal class images for prior preservation loss. If not have enough images, additional images will be"
84
+ " sampled with class_prompt."
85
+ ),
86
+ )
87
+ parser.add_argument(
88
+ "--output_dir",
89
+ type=str,
90
+ default="",
91
+ help="The output directory where the model predictions and checkpoints will be written.",
92
+ )
93
+ parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.")
94
+ parser.add_argument(
95
+ "--resolution",
96
+ type=int,
97
+ default=512,
98
+ help=(
99
+ "The resolution for input images, all the images in the train/validation dataset will be resized to this"
100
+ " resolution"
101
+ ),
102
+ )
103
+ parser.add_argument(
104
+ "--center_crop", action="store_true", help="Whether to center crop images before resizing to resolution"
105
+ )
106
+ parser.add_argument("--train_text_encoder", action="store_true", help="Whether to train the text encoder")
107
+ parser.add_argument(
108
+ "--train_batch_size", type=int, default=4, help="Batch size (per device) for the training dataloader."
109
+ )
110
+ parser.add_argument(
111
+ "--sample_batch_size", type=int, default=4, help="Batch size (per device) for sampling images."
112
+ )
113
+ parser.add_argument("--num_train_epochs", type=int, default=1)
114
+ parser.add_argument(
115
+ "--max_train_steps",
116
+ type=int,
117
+ default=None,
118
+ help="Total number of training steps to perform. If provided, overrides num_train_epochs.",
119
+ )
120
+ parser.add_argument(
121
+ "--gradient_accumulation_steps",
122
+ type=int,
123
+ default=1,
124
+ help="Number of updates steps to accumulate before performing a backward/update pass.",
125
+ )
126
+ parser.add_argument(
127
+ "--gradient_checkpointing",
128
+ action="store_true",
129
+ help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.",
130
+ )
131
+ parser.add_argument(
132
+ "--learning_rate",
133
+ type=float,
134
+ default=5e-6,
135
+ help="Initial learning rate (after the potential warmup period) to use.",
136
+ )
137
+ parser.add_argument(
138
+ "--scale_lr",
139
+ action="store_true",
140
+ default=False,
141
+ help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.",
142
+ )
143
+ parser.add_argument(
144
+ "--lr_scheduler",
145
+ type=str,
146
+ default="constant",
147
+ help=(
148
+ 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",'
149
+ ' "constant", "constant_with_warmup"]'
150
+ ),
151
+ )
152
+ parser.add_argument(
153
+ "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler."
154
+ )
155
+ parser.add_argument(
156
+ "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes."
157
+ )
158
+ parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.")
159
+ parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.")
160
+ parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.")
161
+ parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer")
162
+ parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
163
+ parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.")
164
+ parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.")
165
+ parser.add_argument(
166
+ "--hub_model_id",
167
+ type=str,
168
+ default=None,
169
+ help="The name of the repository to keep in sync with the local `output_dir`.",
170
+ )
171
+ parser.add_argument(
172
+ "--logging_dir",
173
+ type=str,
174
+ default="logs",
175
+ help=(
176
+ "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to"
177
+ " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***."
178
+ ),
179
+ )
180
+ parser.add_argument(
181
+ "--mixed_precision",
182
+ type=str,
183
+ default="no",
184
+ choices=["no", "fp16", "bf16"],
185
+ help=(
186
+ "Whether to use mixed precision. Choose"
187
+ "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
188
+ "and an Nvidia Ampere GPU."
189
+ ),
190
+ )
191
+
192
+ parser.add_argument(
193
+ "--save_n_steps",
194
+ type=int,
195
+ default=1,
196
+ help=("Save the model every n global_steps"),
197
+ )
198
+
199
+
200
+ parser.add_argument(
201
+ "--save_starting_step",
202
+ type=int,
203
+ default=1,
204
+ help=("The step from which it starts saving intermediary checkpoints"),
205
+ )
206
+
207
+ parser.add_argument(
208
+ "--stop_text_encoder_training",
209
+ type=int,
210
+ default=1000000,
211
+ help=("The step at which the text_encoder is no longer trained"),
212
+ )
213
+
214
+
215
+ parser.add_argument(
216
+ "--image_captions_filename",
217
+ action="store_true",
218
+ help="Get captions from filename",
219
+ )
220
+
221
+
222
+ parser.add_argument(
223
+ "--dump_only_text_encoder",
224
+ action="store_true",
225
+ default=False,
226
+ help="Dump only text encoder",
227
+ )
228
+
229
+ parser.add_argument(
230
+ "--train_only_unet",
231
+ action="store_true",
232
+ default=False,
233
+ help="Train only the unet",
234
+ )
235
+
236
+ parser.add_argument(
237
+ "--Session_dir",
238
+ type=str,
239
+ default="",
240
+ help="Current session directory",
241
+ )
242
+
243
+
244
+
245
+
246
+ parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
247
+
248
+ args = parser.parse_args()
249
+ env_local_rank = int(os.environ.get("LOCAL_RANK", -1))
250
+ if env_local_rank != -1 and env_local_rank != args.local_rank:
251
+ args.local_rank = env_local_rank
252
+
253
+ if args.instance_data_dir is None:
254
+ raise ValueError("You must specify a train data directory.")
255
+
256
+ if args.with_prior_preservation:
257
+ if args.class_data_dir is None:
258
+ raise ValueError("You must specify a data directory for class images.")
259
+ if args.class_prompt is None:
260
+ raise ValueError("You must specify prompt for class images.")
261
+
262
+ return args
263
+
264
+
265
+ class DreamBoothDataset(Dataset):
266
+ """
267
+ A dataset to prepare the instance and class images with the prompts for fine-tuning the model.
268
+ It pre-processes the images and the tokenizes prompts.
269
+ """
270
+
271
+ def __init__(
272
+ self,
273
+ instance_data_root,
274
+ instance_prompt,
275
+ tokenizer,
276
+ args,
277
+ class_data_root=None,
278
+ class_prompt=None,
279
+ size=512,
280
+ center_crop=False,
281
+ ):
282
+ self.size = size
283
+ self.center_crop = center_crop
284
+ self.tokenizer = tokenizer
285
+ self.image_captions_filename = None
286
+
287
+ self.instance_data_root = Path(instance_data_root)
288
+ if not self.instance_data_root.exists():
289
+ raise ValueError("Instance images root doesn't exists.")
290
+
291
+ self.instance_images_path = list(Path(instance_data_root).iterdir())
292
+ self.num_instance_images = len(self.instance_images_path)
293
+ self.instance_prompt = instance_prompt
294
+ self._length = self.num_instance_images
295
+
296
+ if args.image_captions_filename:
297
+ self.image_captions_filename = True
298
+
299
+ if class_data_root is not None:
300
+ self.class_data_root = Path(class_data_root)
301
+ self.class_data_root.mkdir(parents=True, exist_ok=True)
302
+ self.class_images_path = list(self.class_data_root.iterdir())
303
+ self.num_class_images = len(self.class_images_path)
304
+ self._length = max(self.num_class_images, self.num_instance_images)
305
+ self.class_prompt = class_prompt
306
+ else:
307
+ self.class_data_root = None
308
+
309
+ self.image_transforms = transforms.Compose(
310
+ [
311
+ transforms.Resize(size, interpolation=transforms.InterpolationMode.BILINEAR),
312
+ transforms.CenterCrop(size) if center_crop else transforms.RandomCrop(size),
313
+ transforms.ToTensor(),
314
+ transforms.Normalize([0.5], [0.5]),
315
+ ]
316
+ )
317
+
318
+ def __len__(self):
319
+ return self._length
320
+
321
+ def __getitem__(self, index):
322
+ example = {}
323
+ path = self.instance_images_path[index % self.num_instance_images]
324
+ instance_image = Image.open(path)
325
+ if not instance_image.mode == "RGB":
326
+ instance_image = instance_image.convert("RGB")
327
+
328
+ instance_prompt = self.instance_prompt
329
+
330
+ if self.image_captions_filename:
331
+ filename = Path(path).stem
332
+ pt=''.join([i for i in filename if not i.isdigit()])
333
+ pt=pt.replace("_"," ")
334
+ pt=pt.replace("(","")
335
+ pt=pt.replace(")","")
336
+ instance_prompt = pt
337
+ sys.stdout.write(" " +instance_prompt+" ")
338
+ sys.stdout.flush()
339
+
340
+
341
+ example["instance_images"] = self.image_transforms(instance_image)
342
+ example["instance_prompt_ids"] = self.tokenizer(
343
+ instance_prompt,
344
+ padding="do_not_pad",
345
+ truncation=True,
346
+ max_length=self.tokenizer.model_max_length,
347
+ ).input_ids
348
+
349
+ if self.class_data_root:
350
+ class_image = Image.open(self.class_images_path[index % self.num_class_images])
351
+ if not class_image.mode == "RGB":
352
+ class_image = class_image.convert("RGB")
353
+ example["class_images"] = self.image_transforms(class_image)
354
+ example["class_prompt_ids"] = self.tokenizer(
355
+ self.class_prompt,
356
+ padding="do_not_pad",
357
+ truncation=True,
358
+ max_length=self.tokenizer.model_max_length,
359
+ ).input_ids
360
+
361
+ return example
362
+
363
+
364
+
365
+ class PromptDataset(Dataset):
366
+ "A simple dataset to prepare the prompts to generate class images on multiple GPUs."
367
+
368
+ def __init__(self, prompt, num_samples):
369
+ self.prompt = prompt
370
+ self.num_samples = num_samples
371
+
372
+ def __len__(self):
373
+ return self.num_samples
374
+
375
+ def __getitem__(self, index):
376
+ example = {}
377
+ example["prompt"] = self.prompt
378
+ example["index"] = index
379
+ return example
380
+
381
+
382
+ def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None):
383
+ if token is None:
384
+ token = HfFolder.get_token()
385
+ if organization is None:
386
+ username = whoami(token)["name"]
387
+ return f"{username}/{model_id}"
388
+ else:
389
+ return f"{organization}/{model_id}"
390
+
391
+
392
+ def main():
393
+ args = parse_args()
394
+ logging_dir = Path(args.output_dir, args.logging_dir)
395
+ i=args.save_starting_step
396
+ accelerator = Accelerator(
397
+ gradient_accumulation_steps=args.gradient_accumulation_steps,
398
+ mixed_precision=args.mixed_precision,
399
+ log_with="tensorboard",
400
+ logging_dir=logging_dir,
401
+ )
402
+
403
+ # Currently, it's not possible to do gradient accumulation when training two models with accelerate.accumulate
404
+ # This will be enabled soon in accelerate. For now, we don't allow gradient accumulation when training two models.
405
+ # TODO (patil-suraj): Remove this check when gradient accumulation with two models is enabled in accelerate.
406
+ if args.train_text_encoder and args.gradient_accumulation_steps > 1 and accelerator.num_processes > 1:
407
+ raise ValueError(
408
+ "Gradient accumulation is not supported when training the text encoder in distributed training. "
409
+ "Please set gradient_accumulation_steps to 1. This feature will be supported in the future."
410
+ )
411
+
412
+ if args.seed is not None:
413
+ set_seed(args.seed)
414
+
415
+ if args.with_prior_preservation:
416
+ class_images_dir = Path(args.class_data_dir)
417
+ if not class_images_dir.exists():
418
+ class_images_dir.mkdir(parents=True)
419
+ cur_class_images = len(list(class_images_dir.iterdir()))
420
+
421
+ if cur_class_images < args.num_class_images:
422
+ torch_dtype = torch.float16 if accelerator.device.type == "cuda" else torch.float32
423
+ pipeline = StableDiffusionPipeline.from_pretrained(
424
+ args.pretrained_model_name_or_path, torch_dtype=torch_dtype
425
+ )
426
+ pipeline.set_progress_bar_config(disable=True)
427
+
428
+ num_new_images = args.num_class_images - cur_class_images
429
+ logger.info(f"Number of class images to sample: {num_new_images}.")
430
+
431
+ sample_dataset = PromptDataset(args.class_prompt, num_new_images)
432
+ sample_dataloader = torch.utils.data.DataLoader(sample_dataset, batch_size=args.sample_batch_size)
433
+
434
+ sample_dataloader = accelerator.prepare(sample_dataloader)
435
+ pipeline.to(accelerator.device)
436
+
437
+ for example in tqdm(
438
+ sample_dataloader, desc="Generating class images", disable=not accelerator.is_local_main_process
439
+ ):
440
+ with torch.autocast("cuda"):
441
+ images = pipeline(example["prompt"]).images
442
+
443
+ for i, image in enumerate(images):
444
+ image.save(class_images_dir / f"{example['index'][i] + cur_class_images}.jpg")
445
+
446
+ del pipeline
447
+ if torch.cuda.is_available():
448
+ torch.cuda.empty_cache()
449
+
450
+ # Handle the repository creation
451
+ if accelerator.is_main_process:
452
+ if args.push_to_hub:
453
+ if args.hub_model_id is None:
454
+ repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token)
455
+ else:
456
+ repo_name = args.hub_model_id
457
+ repo = Repository(args.output_dir, clone_from=repo_name)
458
+
459
+ with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
460
+ if "step_*" not in gitignore:
461
+ gitignore.write("step_*\n")
462
+ if "epoch_*" not in gitignore:
463
+ gitignore.write("epoch_*\n")
464
+ elif args.output_dir is not None:
465
+ os.makedirs(args.output_dir, exist_ok=True)
466
+
467
+ # Load the tokenizer
468
+ if args.tokenizer_name:
469
+ tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name)
470
+ elif args.pretrained_model_name_or_path:
471
+ tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder="tokenizer")
472
+
473
+ # Load models and create wrapper for stable diffusion
474
+ if args.train_only_unet:
475
+ if os.path.exists(str(args.output_dir+"/text_encoder_trained")):
476
+ text_encoder = CLIPTextModel.from_pretrained(args.output_dir, subfolder="text_encoder_trained")
477
+ elif os.path.exists(str(args.output_dir+"/text_encoder")):
478
+ text_encoder = CLIPTextModel.from_pretrained(args.output_dir, subfolder="text_encoder")
479
+ else:
480
+ text_encoder = CLIPTextModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="text_encoder")
481
+ else:
482
+ text_encoder = CLIPTextModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="text_encoder")
483
+ vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path, subfolder="vae")
484
+ unet = UNet2DConditionModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="unet")
485
+
486
+ vae.requires_grad_(False)
487
+ if not args.train_text_encoder:
488
+ text_encoder.requires_grad_(False)
489
+
490
+ if args.gradient_checkpointing:
491
+ unet.enable_gradient_checkpointing()
492
+ if args.train_text_encoder:
493
+ text_encoder.gradient_checkpointing_enable()
494
+
495
+ if args.scale_lr:
496
+ args.learning_rate = (
497
+ args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes
498
+ )
499
+
500
+ # Use 8-bit Adam for lower memory usage or to fine-tune the model in 16GB GPUs
501
+ if args.use_8bit_adam:
502
+ try:
503
+ import bitsandbytes as bnb
504
+ except ImportError:
505
+ raise ImportError(
506
+ "To use 8-bit Adam, please install the bitsandbytes library: `pip install bitsandbytes`."
507
+ )
508
+
509
+ optimizer_class = bnb.optim.AdamW8bit
510
+ else:
511
+ optimizer_class = torch.optim.AdamW
512
+
513
+ params_to_optimize = (
514
+ itertools.chain(unet.parameters(), text_encoder.parameters()) if args.train_text_encoder else unet.parameters()
515
+ )
516
+ optimizer = optimizer_class(
517
+ params_to_optimize,
518
+ lr=args.learning_rate,
519
+ betas=(args.adam_beta1, args.adam_beta2),
520
+ weight_decay=args.adam_weight_decay,
521
+ eps=args.adam_epsilon,
522
+ )
523
+
524
+ noise_scheduler = DDPMScheduler(
525
+ beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000
526
+ )
527
+
528
+ train_dataset = DreamBoothDataset(
529
+ instance_data_root=args.instance_data_dir,
530
+ instance_prompt=args.instance_prompt,
531
+ class_data_root=args.class_data_dir if args.with_prior_preservation else None,
532
+ class_prompt=args.class_prompt,
533
+ tokenizer=tokenizer,
534
+ size=args.resolution,
535
+ center_crop=args.center_crop,
536
+ args=args,
537
+ )
538
+
539
+ def collate_fn(examples):
540
+ input_ids = [example["instance_prompt_ids"] for example in examples]
541
+ pixel_values = [example["instance_images"] for example in examples]
542
+
543
+ # Concat class and instance examples for prior preservation.
544
+ # We do this to avoid doing two forward passes.
545
+ if args.with_prior_preservation:
546
+ input_ids += [example["class_prompt_ids"] for example in examples]
547
+ pixel_values += [example["class_images"] for example in examples]
548
+
549
+ pixel_values = torch.stack(pixel_values)
550
+ pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float()
551
+
552
+ input_ids = tokenizer.pad({"input_ids": input_ids}, padding=True, return_tensors="pt").input_ids
553
+
554
+ batch = {
555
+ "input_ids": input_ids,
556
+ "pixel_values": pixel_values,
557
+ }
558
+ return batch
559
+
560
+ train_dataloader = torch.utils.data.DataLoader(
561
+ train_dataset, batch_size=args.train_batch_size, shuffle=True, collate_fn=collate_fn
562
+ )
563
+
564
+ # Scheduler and math around the number of training steps.
565
+ overrode_max_train_steps = False
566
+ num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
567
+ if args.max_train_steps is None:
568
+ args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
569
+ overrode_max_train_steps = True
570
+
571
+ lr_scheduler = get_scheduler(
572
+ args.lr_scheduler,
573
+ optimizer=optimizer,
574
+ num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps,
575
+ num_training_steps=args.max_train_steps * args.gradient_accumulation_steps,
576
+ )
577
+
578
+ if args.train_text_encoder:
579
+ unet, text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
580
+ unet, text_encoder, optimizer, train_dataloader, lr_scheduler
581
+ )
582
+ else:
583
+ unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
584
+ unet, optimizer, train_dataloader, lr_scheduler
585
+ )
586
+
587
+ weight_dtype = torch.float32
588
+ if args.mixed_precision == "fp16":
589
+ weight_dtype = torch.float16
590
+ elif args.mixed_precision == "bf16":
591
+ weight_dtype = torch.bfloat16
592
+
593
+ # Move text_encode and vae to gpu.
594
+ # For mixed precision training we cast the text_encoder and vae weights to half-precision
595
+ # as these models are only used for inference, keeping weights in full precision is not required.
596
+ vae.to(accelerator.device, dtype=weight_dtype)
597
+ if not args.train_text_encoder:
598
+ text_encoder.to(accelerator.device, dtype=weight_dtype)
599
+
600
+ # We need to recalculate our total training steps as the size of the training dataloader may have changed.
601
+ num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
602
+ if overrode_max_train_steps:
603
+ args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
604
+ # Afterwards we recalculate our number of training epochs
605
+ args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
606
+
607
+ # We need to initialize the trackers we use, and also store our configuration.
608
+ # The trackers initializes automatically on the main process.
609
+ if accelerator.is_main_process:
610
+ accelerator.init_trackers("dreambooth", config=vars(args))
611
+
612
+ def bar(prg):
613
+ br='|'+'█' * prg + ' ' * (25-prg)+'|'
614
+ return br
615
+
616
+ # Train!
617
+ total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
618
+
619
+ logger.info("***** Running training *****")
620
+ logger.info(f" Num examples = {len(train_dataset)}")
621
+ logger.info(f" Num batches each epoch = {len(train_dataloader)}")
622
+ logger.info(f" Num Epochs = {args.num_train_epochs}")
623
+ logger.info(f" Instantaneous batch size per device = {args.train_batch_size}")
624
+ logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
625
+ logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}")
626
+ logger.info(f" Total optimization steps = {args.max_train_steps}")
627
+ # Only show the progress bar once on each machine.
628
+ progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)
629
+ global_step = 0
630
+
631
+ for epoch in range(args.num_train_epochs):
632
+ unet.train()
633
+ if args.train_text_encoder:
634
+ text_encoder.train()
635
+ for step, batch in enumerate(train_dataloader):
636
+ with accelerator.accumulate(unet):
637
+ # Convert images to latent space
638
+ latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample()
639
+ latents = latents * 0.18215
640
+
641
+ # Sample noise that we'll add to the latents
642
+ noise = torch.randn_like(latents)
643
+ bsz = latents.shape[0]
644
+ # Sample a random timestep for each image
645
+ timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device)
646
+ timesteps = timesteps.long()
647
+
648
+ # Add noise to the latents according to the noise magnitude at each timestep
649
+ # (this is the forward diffusion process)
650
+ noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
651
+
652
+ # Get the text embedding for conditioning
653
+ encoder_hidden_states = text_encoder(batch["input_ids"])[0]
654
+
655
+ # Predict the noise residual
656
+ noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample
657
+
658
+ if args.with_prior_preservation:
659
+ # Chunk the noise and noise_pred into two parts and compute the loss on each part separately.
660
+ noise_pred, noise_pred_prior = torch.chunk(noise_pred, 2, dim=0)
661
+ noise, noise_prior = torch.chunk(noise, 2, dim=0)
662
+
663
+ # Compute instance loss
664
+ loss = F.mse_loss(noise_pred.float(), noise.float(), reduction="none").mean([1, 2, 3]).mean()
665
+
666
+ # Compute prior loss
667
+ prior_loss = F.mse_loss(noise_pred_prior.float(), noise_prior.float(), reduction="mean")
668
+
669
+ # Add the prior loss to the instance loss.
670
+ loss = loss + args.prior_loss_weight * prior_loss
671
+ else:
672
+ loss = F.mse_loss(noise_pred.float(), noise.float(), reduction="mean")
673
+
674
+ accelerator.backward(loss)
675
+ if accelerator.sync_gradients:
676
+ params_to_clip = (
677
+ itertools.chain(unet.parameters(), text_encoder.parameters())
678
+ if args.train_text_encoder
679
+ else unet.parameters()
680
+ )
681
+ accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
682
+ optimizer.step()
683
+ lr_scheduler.step()
684
+ optimizer.zero_grad()
685
+
686
+ # Checks if the accelerator has performed an optimization step behind the scenes
687
+ if accelerator.sync_gradients:
688
+ progress_bar.update(1)
689
+ global_step += 1
690
+
691
+ fll=round((global_step*100)/args.max_train_steps)
692
+ fll=round(fll/4)
693
+ pr=bar(fll)
694
+
695
+ logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]}
696
+ progress_bar.set_postfix(**logs)
697
+ progress_bar.set_description_str("Progress:"+pr)
698
+ accelerator.log(logs, step=global_step)
699
+
700
+ if global_step >= args.max_train_steps:
701
+ break
702
+
703
+ if args.train_text_encoder and global_step == args.stop_text_encoder_training and global_step >= 30:
704
+ if accelerator.is_main_process:
705
+ print(" " +" Freezing the text_encoder ..."+" ")
706
+ frz_dir=args.output_dir + "/text_encoder_frozen"
707
+ if os.path.exists(frz_dir):
708
+ subprocess.call('rm -r '+ frz_dir, shell=True)
709
+ os.mkdir(frz_dir)
710
+ pipeline = StableDiffusionPipeline.from_pretrained(
711
+ args.pretrained_model_name_or_path,
712
+ unet=accelerator.unwrap_model(unet),
713
+ text_encoder=accelerator.unwrap_model(text_encoder),
714
+ )
715
+ pipeline.text_encoder.save_pretrained(frz_dir)
716
+
717
+ if args.save_n_steps >= 200:
718
+ if global_step < args.max_train_steps-100 and global_step+1==i:
719
+ ckpt_name = "_step_" + str(global_step+1)
720
+ save_dir = Path(args.output_dir+ckpt_name)
721
+ save_dir=str(save_dir)
722
+ save_dir=save_dir.replace(" ", "_")
723
+ if not os.path.exists(save_dir):
724
+ os.mkdir(save_dir)
725
+ inst=save_dir[16:]
726
+ inst=inst.replace(" ", "_")
727
+ print(" SAVING CHECKPOINT: "+args.Session_dir+"/"+inst+".ckpt")
728
+ # Create the pipeline using the trained modules and save it.
729
+ if accelerator.is_main_process:
730
+ pipeline = StableDiffusionPipeline.from_pretrained(
731
+ args.pretrained_model_name_or_path,
732
+ unet=accelerator.unwrap_model(unet),
733
+ text_encoder=accelerator.unwrap_model(text_encoder),
734
+ )
735
+ pipeline.save_pretrained(save_dir)
736
+ frz_dir=args.output_dir + "/text_encoder_frozen"
737
+ if args.train_text_encoder and os.path.exists(frz_dir):
738
+ subprocess.call('rm -r '+save_dir+'/text_encoder/*.*', shell=True)
739
+ subprocess.call('cp -f '+frz_dir +'/*.* '+ save_dir+'/text_encoder', shell=True)
740
+ chkpth=args.Session_dir+"/"+inst+".ckpt"
741
+ subprocess.call('python /content/diffusers/scripts/convert_diffusers_to_original_stable_diffusion.py --model_path ' + save_dir + ' --checkpoint_path ' + chkpth + ' --half', shell=True)
742
+ i=i+args.save_n_steps
743
+
744
+ accelerator.wait_for_everyone()
745
+
746
+ # Create the pipeline using using the trained modules and save it.
747
+ if accelerator.is_main_process:
748
+ if args.dump_only_text_encoder:
749
+ txt_dir=args.output_dir + "/text_encoder_trained"
750
+ if not os.path.exists(txt_dir):
751
+ os.mkdir(txt_dir)
752
+ pipeline = StableDiffusionPipeline.from_pretrained(
753
+ args.pretrained_model_name_or_path,
754
+ unet=accelerator.unwrap_model(unet),
755
+ text_encoder=accelerator.unwrap_model(text_encoder),
756
+ )
757
+ pipeline.text_encoder.save_pretrained(txt_dir)
758
+
759
+ elif args.train_only_unet:
760
+ pipeline = StableDiffusionPipeline.from_pretrained(
761
+ args.pretrained_model_name_or_path,
762
+ unet=accelerator.unwrap_model(unet),
763
+ text_encoder=accelerator.unwrap_model(text_encoder),
764
+ )
765
+ pipeline.save_pretrained(args.output_dir)
766
+ txt_dir=args.output_dir + "/text_encoder_trained"
767
+ subprocess.call('rm -r '+txt_dir, shell=True)
768
+
769
+ else:
770
+ pipeline = StableDiffusionPipeline.from_pretrained(
771
+ args.pretrained_model_name_or_path,
772
+ unet=accelerator.unwrap_model(unet),
773
+ text_encoder=accelerator.unwrap_model(text_encoder),
774
+ )
775
+ frz_dir=args.output_dir + "/text_encoder_frozen"
776
+ pipeline.save_pretrained(args.output_dir)
777
+ if args.train_text_encoder and os.path.exists(frz_dir):
778
+ subprocess.call('mv -f '+frz_dir +'/*.* '+ args.output_dir+'/text_encoder', shell=True)
779
+ subprocess.call('rm -r '+ frz_dir, shell=True)
780
+
781
+ if args.push_to_hub:
782
+ repo.push_to_hub(commit_message="End of training", blocking=False, auto_lfs_prune=True)
783
+
784
+ accelerator.end_training()
785
+
786
+ if __name__ == "__main__":
787
+ main()