| { | |
| "experiment_key": "base", | |
| "save_safetensors": true, | |
| "max_shard_size": "10GB", | |
| "local_rank": 1, | |
| "use_gradient_checkpointing": true, | |
| "trainer_key": "lm", | |
| "force_fp32": false, | |
| "force_fp16": false, | |
| "from_gptq": false, | |
| "huggingface_hub_token": null, | |
| "deepspeed_stage": 2, | |
| "deepspeed_config_path": null, | |
| "fsdp_strategy": "", | |
| "fsdp_offload": true, | |
| "seed": 42, | |
| "stabilize": true, | |
| "path_to_env_file": "./.env", | |
| "prepare_dataset": true, | |
| "lora_hub_model_id": null, | |
| "lora_model_local_path": null, | |
| "fused_model_local_path": null, | |
| "quantization_dataset_id": null, | |
| "quantization_max_samples": 1024, | |
| "quantized_model_path": "./quantized_model/", | |
| "quantized_hub_model_id": null, | |
| "quantized_hub_private_repo": null, | |
| "dataset_key": "desc2json", | |
| "train_local_path_to_data": "./train.jsonl", | |
| "eval_local_path_to_data": "./eval.jsonl", | |
| "shuffle": true, | |
| "max_eval_samples": 1000, | |
| "add_eval_to_train_if_no_path": false, | |
| "tokenizer_name_or_path": null, | |
| "tokenizer_use_fast": null, | |
| "tokenizer_padding_side": null, | |
| "collator_key": "completion", | |
| "max_length": 2048, | |
| "model_name_or_path": "mistralai/Mistral-7B-v0.1", | |
| "push_to_hub_bos_add_bos_token": false, | |
| "use_flash_attention_2": false, | |
| "trust_remote_code": true, | |
| "device_map": null, | |
| "prepare_model_for_kbit_training": true, | |
| "load_in_8bit": false, | |
| "load_in_4bit": true, | |
| "llm_int8_threshold": 6.0, | |
| "llm_int8_has_fp16_weight": true, | |
| "bnb_4bit_use_double_quant": true, | |
| "bnb_4bit_quant_type": "nf4", | |
| "bnb_quantize_after_model_init": false, | |
| "gptq_bits": 4, | |
| "gptq_group_size": 128, | |
| "gptq_disable_exllama": true, | |
| "apply_lora": true, | |
| "lora_rank": 64, | |
| "lora_alpha": 32, | |
| "lora_dropout": 0.1, | |
| "raw_lora_target_modules": "all", | |
| "output_dir": "./outputs/", | |
| "per_device_train_batch_size": 8, | |
| "do_eval": true, | |
| "per_device_eval_batch_size": null, | |
| "gradient_accumulation_steps": 2, | |
| "eval_accumulation_steps": null, | |
| "eval_delay": 0, | |
| "eval_steps": 300, | |
| "warmup_steps": 100, | |
| "max_steps": null, | |
| "num_train_epochs": 5, | |
| "learning_rate": 0.0002, | |
| "max_grad_norm": 1.0, | |
| "weight_decay": 0.001, | |
| "label_smoothing_factor": 0.1, | |
| "logging_steps": 1, | |
| "save_steps": 300, | |
| "save_total_limit": 0, | |
| "optim": "paged_adamw_8bit", | |
| "push_to_hub": true, | |
| "hub_model_id": "BobaZooba/WGPT-LoRA", | |
| "hub_private_repo": false, | |
| "report_to_wandb": true, | |
| "wandb_api_key": null, | |
| "wandb_project": null, | |
| "wandb_entity": null | |
| } |