| wandb_project: joy-caption-1 | |
| device_batch_size: 2 | |
| batch_size: 256 | |
| learning_rate: 0.0002 | |
| warmup_samples: 18000 | |
| max_samples: 500000 | |
| save_every: 50000 | |
| test_every: 50000 | |
| use_amp: true | |
| grad_scaler: true | |
| lr_scheduler_type: cosine | |
| min_lr_ratio: 0.0 | |
| allow_tf32: true | |
| seed: 69 | |
| num_workers: 8 | |
| optimizer_type: adamw | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.999 | |
| adam_eps: 1.0e-08 | |
| adam_weight_decay: 0.0 | |
| clip_grad_norm: 1.0 | |
| dataset: fancyfeast/joy-captioning-20240917a | |
| clip_model: google/siglip-so400m-patch14-384 | |
| text_model: meta-llama/Meta-Llama-3.1-8B | |
| resume: null | |
| gradient_checkpointing: false | |
| test_size: 2048 | |
| grad_scaler_init: 65536.0 | |
| max_caption_length: 257 | |
| num_image_tokens: 32 | |
| adapter_type: mlp | |
| text_model_dtype: bfloat16 | |
| pre_test: false | |
| train_image_model: true | |
| image_model_lr: null | |
| train_lora: true | |
| lora_r: 64 | |
| lora_alpha: 16 | |
| lora_dropout: 0.1 | |