| { | |
| "doc": { | |
| "optimizer": "adamw, adamw8bit, lion", | |
| "optimizer_desc": "'adamw' in standard 32bit, 'adamw8bit' is bitsandbytes, 'lion' is lucidrains", | |
| "lr": "learning rate, if null wil use CLI or main JSON config value", | |
| "betas": "exponential decay rates for the moment estimates", | |
| "epsilon": "value added to denominator for numerical stability, unused for lion", | |
| "weight_decay": "weight decay (L2 penalty)", | |
| "text_encoder_lr_scale": "scale the text encoder LR relative to the Unet LR. for example, if `lr` is 2e-6 and `text_encoder_lr_scale` is 0.5, the text encoder's LR will be set to `1e-6`." | |
| }, | |
| "optimizer": "adamw8bit", | |
| "lr": 1e-6, | |
| "betas": [0.9, 0.999], | |
| "epsilon": 1e-8, | |
| "weight_decay": 0.010, | |
| "text_encoder_lr_scale": 1.0 | |
| } | |