training parameters

Training Parameters/events.out.tfevents.1684502811.irakli-pc.22776.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18f36d9dccfa3ec97d33f6d5fbee74c807f042b57c094a5c88fb5d70fbc463e9
+size 69271642

Training Parameters/optimizer.json

@@ -0,0 +1,17 @@
+{
+    "doc": {
+        "optimizer": "adamw, adamw8bit, lion",
+        "optimizer_desc": "'adamw' in standard 32bit, 'adamw8bit' is bitsandbytes, 'lion' is lucidrains",
+        "lr": "learning rate, if null wil use CLI or main JSON config value",
+        "betas": "exponential decay rates for the moment estimates",
+        "epsilon": "value added to denominator for numerical stability, unused for lion",
+        "weight_decay": "weight decay (L2 penalty)",
+        "text_encoder_lr_scale": "scale the text encoder LR relative to the Unet LR. for example, if `lr` is 2e-6 and `text_encoder_lr_scale` is 0.5, the text encoder's LR will be set to `1e-6`."
+    },
+    "optimizer": "adamw8bit",
+    "lr": 1e-6,
+    "betas": [0.9, 0.999],
+    "epsilon": 1e-8,
+    "weight_decay": 0.010,
+    "text_encoder_lr_scale": 1.0
+}

Training Parameters/train.json

@@ -0,0 +1,49 @@
+{
+  "amp": false,
+  "batch_size": 4,
+  "ckpt_every_n_minutes": null,
+  "clip_grad_norm": null,
+  "clip_skip": 0,
+  "cond_dropout": 0.05,
+  "data_root": "D:\\ED2\\EveryDream2trainer\\input\\vokda_v3_training",
+  "disable_textenc_training": false,
+  "disable_xformers": false,
+  "flip_p": 0.0,
+  "gpuid": 0,
+  "gradient_checkpointing": true,
+  "grad_accum": 1,
+  "logdir": "logs",
+  "log_step": 1,
+  "lowvram": false,
+  "lr": 1.5e-07,
+  "lr_decay_steps": 0,
+  "lr_scheduler": "constant",
+  "lr_warmup_steps": 0,
+  "max_epochs": 100,
+  "notebook": false,
+  "optimizer_config": "optimizer.json",
+  "project_name": "vodka_v3",
+  "resolution": 512,
+  "resume_ckpt": "sd_v1-5_vae",
+  "run_name": null,
+  "sample_prompts": "sample_prompts.txt",
+  "sample_steps": 5000000,
+  "save_ckpt_dir": null,
+  "save_ckpts_from_n_epochs": 0,
+  "save_every_n_epochs": 25,
+  "save_optimizer": false,
+  "scale_lr": false,
+  "seed": 555,
+  "shuffle_tags": false,
+  "validation_config": "validation_default.json",
+  "wandb": false,
+  "write_schedule": false,
+  "rated_dataset": false,
+  "rated_dataset_target_dropout_percent": 50,
+  "zero_frequency_noise_ratio": 0.00,
+  "save_full_precision": false,
+  "disable_unet_training": false,
+  "rated_dataset_target_dropout_rate": 50,
+  "disable_amp": false,
+  "useadam8bit": false
+}

Training Parameters/validation_default.json

@@ -0,0 +1,20 @@
+{
+    "documentation": {
+        "validate_training": "If true, validate the training using a separate set of image/caption pairs, and log the results as `loss/val`. The curve will trend downwards as the model trains, then flatten and start to trend upwards as effective training finishes and the model begins to overfit the training data. Very useful for preventing overfitting, for checking if your learning rate is too low or too high, and for deciding when to stop training.",
+        "val_split_mode": "Either 'automatic' or 'manual', ignored if validate_training is false. 'automatic' val_split_mode picks a random subset of the training set (the number of items is controlled by val_split_proportion) and removes them from training to use as a validation set. 'manual' val_split_mode lets you provide your own folder of validation items (images+captions), specified using 'val_data_root'.",
+        "val_split_proportion": "For 'automatic' val_split_mode, how much of the train dataset that should be removed to use for validation. Typical values are 0.15-0.2 (15-20% of the total dataset). Higher is more accurate but slower.",
+        "val_data_root": "For 'manual' val_split_mode, the path to a folder containing validation items.",
+        "stabilize_training_loss": "If true, stabilize the train loss curves for `loss/epoch` and `loss/log step` by re-calculating training loss with a fixed random seed, and log the results as `loss/train-stabilized`. This more clearly shows the training progress, but it is not enough alone to tell you if you're overfitting.",
+        "stabilize_split_proportion": "For stabilize_training_loss, the proportion of the train dataset to overlap for stabilizing the train loss graph. Typical values are 0.15-0.2 (15-20% of the total dataset). Higher is more accurate but slower.",
+        "every_n_epochs": "How often to run validation (1=every epoch).",
+        "seed": "The seed to use when running validation and stabilization passes."
+    },
+    "validate_training": true,
+    "val_split_mode": "automatic",
+    "val_data_root": null,
+    "val_split_proportion": 0.15,
+    "stabilize_training_loss": true,
+    "stabilize_split_proportion": 0.15,
+    "every_n_epochs": 1,
+    "seed": 555
+}

Training Parameters/vodka_v3-20230519-212641.log

@@ -0,0 +1,54 @@
+05/19/2023 09:26:41 PM  Seed: 555
+05/19/2023 09:26:41 PM  unet attention_head_dim: 8
+05/19/2023 09:26:41 PM Inferred yaml: v1-inference.yaml, attn: sd1, prediction_type: epsilon
+05/19/2023 09:26:50 PM Enabled xformers
+05/19/2023 09:26:51 PM Overriding LR from optimizer config with main config/cli LR setting: 1.5e-07
+05/19/2023 09:26:51 PM  * Loaded optimizer args from optimizer.json *
+05/19/2023 09:26:51 PM  * Training Text and Unet *
+05/19/2023 09:26:51 PM  * Optimizer: AdamW8bit *
+05/19/2023 09:26:51 PM     unet lr: 1.5e-07, text encoder lr: 1.5e-07, betas: [0.9, 0.999], epsilon: 1e-08, weight_decay: 0.01 *
+05/19/2023 09:26:51 PM * DLMA resolution 512, buckets: [[512, 512], [576, 448], [448, 576], [640, 384], [384, 640], [768, 320], [320, 768], [896, 256], [256, 896], [1024, 256], [256, 1024]]
+05/19/2023 09:26:51 PM  Preloading images...
+05/19/2023 09:27:50 PM  * Removed 1629 images from the training set to use for validation
+05/19/2023 09:27:50 PM  * DLMA initialized with 1629 images.
+05/19/2023 09:27:50 PM  ** Dataset 'val': 410 batches, num_images: 1640, batch_size: 4
+05/19/2023 09:27:50 PM  * DLMA initialized with 1385 images.
+05/19/2023 09:27:50 PM  ** Dataset 'stabilize-train': 348 batches, num_images: 1392, batch_size: 4
+05/19/2023 09:27:50 PM  * DLMA initialized with 9227 images.
+05/19/2023 09:27:51 PM  ** Dataset 'train': 2311 batches, num_images: 9244, batch_size: 4
+05/19/2023 09:27:51 PM  Pretraining GPU Memory: 5971 / 24576 MB
+05/19/2023 09:27:51 PM  saving ckpts every 1000000000.0 minutes
+05/19/2023 09:27:51 PM  saving ckpts every 25 epochs
+05/19/2023 09:27:51 PM  unet device: cuda:0, precision: torch.float32, training: True
+05/19/2023 09:27:51 PM  text_encoder device: cuda:0, precision: torch.float32, training: True
+05/19/2023 09:27:51 PM  vae device: cuda:0, precision: torch.float16, training: False
+05/19/2023 09:27:51 PM  scheduler: <class 'diffusers.schedulers.scheduling_ddpm.DDPMScheduler'>
+05/19/2023 09:27:51 PM  Project name: vodka_v3
+05/19/2023 09:27:51 PM  grad_accum: 1
+05/19/2023 09:27:51 PM  batch_size: 4
+05/19/2023 09:27:51 PM  epoch_len: 2311
+05/19/2023 09:27:51 PM  Grad scaler enabled: True (amp mode)
+05/20/2023 10:37:37 AM  Saving model, 25 epochs at step 57775
+05/20/2023 10:37:37 AM  * Saving diffusers model to logs\vodka_v3_20230519-212641/ckpts/vodka_v3-ep25-gs57775
+05/20/2023 10:37:50 AM  * Saving SD model to .\vodka_v3-ep25-gs57775.ckpt
+05/20/2023 11:50:02 PM  Saving model, 25 epochs at step 115550
+05/20/2023 11:50:02 PM  * Saving diffusers model to logs\vodka_v3_20230519-212641/ckpts/vodka_v3-ep50-gs115550
+05/20/2023 11:50:20 PM  * Saving SD model to .\vodka_v3-ep50-gs115550.ckpt
+05/21/2023 09:22:18 AM Validation loss shows diverging
+05/21/2023 09:53:39 AM Validation loss shows diverging
+05/21/2023 01:03:47 PM  Saving model, 25 epochs at step 173325
+05/21/2023 01:03:47 PM  * Saving diffusers model to logs\vodka_v3_20230519-212641/ckpts/vodka_v3-ep75-gs173325
+05/21/2023 01:03:59 PM  * Saving SD model to .\vodka_v3-ep75-gs173325.ckpt
+05/21/2023 07:23:34 PM Validation loss shows diverging
+05/21/2023 08:26:50 PM Validation loss shows diverging
+05/22/2023 01:14:39 AM Validation loss shows diverging
+05/22/2023 01:46:05 AM Validation loss shows diverging
+05/22/2023 02:17:23 AM Validation loss shows diverging
+05/22/2023 02:17:23 AM  * Saving diffusers model to logs\vodka_v3_20230519-212641/ckpts/last-vodka_v3-ep99-gs231100
+05/22/2023 02:17:32 AM  * Saving SD model to .\last-vodka_v3-ep99-gs231100.ckpt
+05/22/2023 02:17:43 AM Training complete
+05/22/2023 02:17:43 AM Total training time took 3169.86 minutes, total steps: 231100
+05/22/2023 02:17:43 AM Average epoch time: 28.11 minutes
+05/22/2023 02:17:43 AM  ***************************
+05/22/2023 02:17:43 AM  **** Finished training ****
+05/22/2023 02:17:43 AM  ***************************