| bnb_cfgs: | |
| bnb_4bit_compute_dtype: float16 | |
| bnb_4bit_quant_type: nf4 | |
| bnb_4bit_use_double_quant: true | |
| load_in_4bit: true | |
| load_in_8bit: false | |
| use_bnb: false | |
| data_cfgs: | |
| eval_data_files: null | |
| eval_datasets: null | |
| eval_optional_args: [] | |
| eval_size: null | |
| eval_split: null | |
| eval_subset: null | |
| eval_template: null | |
| ptx_data_files: flux_ptx_8k_t2i.pt | |
| ptx_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs | |
| ptx_optional_args: [] | |
| ptx_size: null | |
| ptx_split: null | |
| ptx_subset: null | |
| ptx_template: Chameleon_preference | |
| train_data_files: t2i_llf_prompt_only_tokenize.pt | |
| train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs | |
| train_optional_args: [] | |
| train_size: 5000 | |
| train_split: null | |
| train_subset: null | |
| train_template: spavl_ti2ti | |
| logger_cfgs: | |
| cache_dir: null | |
| log_project: align-anything | |
| log_run_name: ppo | |
| log_type: wandb | |
| output_dir: ../outputs/ppo_t2i_llf_1018 | |
| save_interval: 30.0 | |
| lora_cfgs: | |
| inference_mode: false | |
| lora_alpha: 16 | |
| lora_dropout: 0.1 | |
| r: 16 | |
| save_full_model: true | |
| target_modules: | |
| - q_proj | |
| - v_proj | |
| task_type: TaskType.CAUSAL_LM | |
| use_lora: false | |
| model_cfgs: | |
| actor_model_name_or_path: /data/align-anything/hantao/models/0830_4k_sft_flux | |
| model_max_length: 2048 | |
| repetition_penalty: 1.0 | |
| reward_critic_model_name_or_path: /data/align-anything/hantao/align-anything/outputs/rm_t2i_llf_1017 | |
| reward_model_name_or_path: /data/align-anything/hantao/align-anything/outputs/rm_t2i_llf_1017 | |
| temperature: 1.0 | |
| top_p: 1.0 | |
| trust_remote_code: true | |
| special_tokens: null | |
| train_cfgs: | |
| actor_gradient_checkpointing: true | |
| actor_lr: 1.0e-05 | |
| actor_lr_scheduler_type: cosine | |
| actor_lr_warmup_ratio: 0.03 | |
| actor_weight_decay: 0.01 | |
| adam_betas: | |
| - 0.9 | |
| - 0.95 | |
| bf16: true | |
| clip_range_ratio: 0.2 | |
| clip_range_score: 50.0 | |
| clip_range_value: 5.0 | |
| critic_gradient_checkpointing: true | |
| critic_lr: 5.0e-06 | |
| critic_lr_scheduler_type: constant | |
| critic_lr_warmup_ratio: 0.03 | |
| critic_weight_decay: 0.0 | |
| ds_cfgs: ds_z3_config.json | |
| epochs: 3 | |
| eval_interval: 10 | |
| eval_strategy: epoch | |
| fp16: false | |
| freeze_language_model: true | |
| freeze_mm_proj: true | |
| freeze_vision_tower: false | |
| gae_lambda: 0.95 | |
| gamma: 1.0 | |
| gradient_accumulation_steps: 2 | |
| kl_coeff: 0.02 | |
| normalize_reward: false | |
| per_device_eval_batch_size: 8 | |
| per_device_prompt_batch_size: 8 | |
| per_device_train_batch_size: 8 | |
| ptx_coeff: 16.0 | |
| seed: 42 | |
| update_iters: 1 | |