|
--- |
|
base_model: google/gemma-2b |
|
pipeline_tag: text-generation |
|
library_name: peft |
|
--- |
|
|
|
# Model Card for Model ID |
|
|
|
Fine-tuned on python |
|
|
|
|
|
## Model Details |
|
|
|
### Model Description |
|
|
|
|
|
### Model Sources [optional] |
|
Gemma-2b trained on python-oasst dataset |
|
|
|
## Uses |
|
|
|
|
|
## Training Details |
|
|
|
### Training Data |
|
|
|
{ |
|
"_timestamp": 1711018613.433522, |
|
"train/grad_norm": 0.1240904619429259, |
|
"train/global_step": 232, |
|
"eval/steps_per_second": 2.894, |
|
"_step": 232, |
|
"_runtime": 2545.4226660728455, |
|
"eval/loss": 1.189491629600525, |
|
"eval/runtime": 1805.8574, |
|
"train/learning_rate": 0.000014800637958532697, |
|
"eval/samples_per_second": 23.152, |
|
"_wandb.runtime": 2547, |
|
"train/loss": 1.0436, |
|
"train/epoch": 0.01 |
|
} |
|
|
|
### Results |
|
|
|
|
|
#### Summary |
|
|
|
|
|
|
|
|
|
## Technical Specifications [optional] |
|
{ |
|
"bf16": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"fp16": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"fsdp": { |
|
"desc": null, |
|
"value": [] |
|
}, |
|
"seed": { |
|
"desc": null, |
|
"value": 42 |
|
}, |
|
"tf32": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"debug": { |
|
"desc": null, |
|
"value": [] |
|
}, |
|
"optim": { |
|
"desc": null, |
|
"value": "adamw_bnb_8bit" |
|
}, |
|
"qlora": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"top_k": { |
|
"desc": null, |
|
"value": 50 |
|
}, |
|
"top_p": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"_wandb": { |
|
"desc": null, |
|
"value": { |
|
"m": [ |
|
{ |
|
"1": "train/global_step", |
|
"6": [ |
|
3 |
|
] |
|
}, |
|
{ |
|
"1": "train/loss", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "train/grad_norm", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "train/learning_rate", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "train/epoch", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "eval/loss", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "eval/runtime", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "eval/samples_per_second", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
}, |
|
{ |
|
"1": "eval/steps_per_second", |
|
"5": 1, |
|
"6": [ |
|
1 |
|
] |
|
} |
|
], |
|
"t": { |
|
"1": [ |
|
1, |
|
5, |
|
11, |
|
49, |
|
51, |
|
53, |
|
55, |
|
71, |
|
84, |
|
98, |
|
99, |
|
100, |
|
105 |
|
], |
|
"2": [ |
|
1, |
|
5, |
|
11, |
|
49, |
|
51, |
|
53, |
|
55, |
|
71, |
|
84, |
|
98, |
|
99, |
|
100, |
|
105 |
|
], |
|
"3": [ |
|
3, |
|
7, |
|
23 |
|
], |
|
"4": "3.10.13", |
|
"5": "0.16.4", |
|
"6": "4.39.0.dev0", |
|
"8": [ |
|
5 |
|
], |
|
"9": { |
|
"1": "transformers_trainer" |
|
}, |
|
"13": "linux-x86_64" |
|
}, |
|
"framework": "huggingface", |
|
"start_time": 1711016068, |
|
"cli_version": "0.16.4", |
|
"is_jupyter_run": false, |
|
"python_version": "3.10.13", |
|
"is_kaggle_kernel": false, |
|
"huggingface_version": "4.39.0.dev0" |
|
} |
|
}, |
|
"prefix": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"do_eval": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"no_cuda": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"use_cpu": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"do_train": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"head_dim": { |
|
"desc": null, |
|
"value": 256 |
|
}, |
|
"id2label": { |
|
"desc": null, |
|
"value": { |
|
"0": "LABEL_0", |
|
"1": "LABEL_1" |
|
} |
|
}, |
|
"label2id": { |
|
"desc": null, |
|
"value": { |
|
"LABEL_0": 0, |
|
"LABEL_1": 1 |
|
} |
|
}, |
|
"run_name": { |
|
"desc": null, |
|
"value": "./out" |
|
}, |
|
"use_ipex": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"adafactor": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"data_seed": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"deepspeed": { |
|
"desc": null, |
|
"value": "deepspeed_configs/zero1.json" |
|
}, |
|
"do_sample": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"hub_token": { |
|
"desc": null, |
|
}, |
|
"log_level": { |
|
"desc": null, |
|
"value": "passive" |
|
}, |
|
"max_steps": { |
|
"desc": null, |
|
"value": -1 |
|
}, |
|
"num_beams": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"ray_scope": { |
|
"desc": null, |
|
"value": "last" |
|
}, |
|
"report_to": { |
|
"desc": null, |
|
"value": [ |
|
"wandb" |
|
] |
|
}, |
|
"typical_p": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"use_cache": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"adam_beta1": { |
|
"desc": null, |
|
"value": 0.9 |
|
}, |
|
"adam_beta2": { |
|
"desc": null, |
|
"value": 0.999 |
|
}, |
|
"do_predict": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"eval_delay": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"eval_steps": { |
|
"desc": null, |
|
"value": 0.03125 |
|
}, |
|
"hidden_act": { |
|
"desc": null, |
|
"value": "gelu" |
|
}, |
|
"is_decoder": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"local_rank": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"max_length": { |
|
"desc": null, |
|
"value": 20 |
|
}, |
|
"min_length": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"model_type": { |
|
"desc": null, |
|
"value": "gemma" |
|
}, |
|
"optim_args": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"orpo_alpha": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"output_dir": { |
|
"desc": null, |
|
"value": "./out" |
|
}, |
|
"past_index": { |
|
"desc": null, |
|
"value": -1 |
|
}, |
|
"rope_theta": { |
|
"desc": null, |
|
"value": 10000 |
|
}, |
|
"save_steps": { |
|
"desc": null, |
|
"value": 0.125 |
|
}, |
|
"vocab_size": { |
|
"desc": null, |
|
"value": 256000 |
|
}, |
|
"bench_split": { |
|
"desc": null, |
|
"value": "eval" |
|
}, |
|
"ddp_backend": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"ddp_timeout": { |
|
"desc": null, |
|
"value": 1800 |
|
}, |
|
"fsdp_config": { |
|
"desc": null, |
|
"value": { |
|
"xla": false, |
|
"xla_fsdp_v2": false, |
|
"min_num_params": 0, |
|
"xla_fsdp_grad_ckpt": false |
|
} |
|
}, |
|
"hidden_size": { |
|
"desc": null, |
|
"value": 2048 |
|
}, |
|
"label_names": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"logging_dir": { |
|
"desc": null, |
|
"value": "./out/runs/Mar21_10-14-24_8205afe3ecd2" |
|
}, |
|
"pretraining": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"push_to_hub": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"return_dict": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"temperature": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"torch_dtype": { |
|
"desc": null, |
|
"value": "bfloat16" |
|
}, |
|
"torchdynamo": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"torchscript": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"adam_epsilon": { |
|
"desc": null, |
|
"value": 1e-8 |
|
}, |
|
"bos_token_id": { |
|
"desc": null, |
|
"value": 2 |
|
}, |
|
"disable_tqdm": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"eos_token_id": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"fp16_backend": { |
|
"desc": null, |
|
"value": "auto" |
|
}, |
|
"hub_model_id": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"hub_strategy": { |
|
"desc": null, |
|
"value": "every_save" |
|
}, |
|
"pad_token_id": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"problem_type": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"pruned_heads": { |
|
"desc": null, |
|
"value": {} |
|
}, |
|
"relora_steps": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"rms_norm_eps": { |
|
"desc": null, |
|
"value": 0.000001 |
|
}, |
|
"rope_scaling": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"sep_token_id": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"use_bfloat16": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"warmup_ratio": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"warmup_steps": { |
|
"desc": null, |
|
"value": 3135 |
|
}, |
|
"weight_decay": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"_name_or_path": { |
|
"desc": null, |
|
"value": "dvdmrs09/gemma2b-train" |
|
}, |
|
"architectures": { |
|
"desc": null, |
|
"value": [ |
|
"GemmaForCausalLM" |
|
] |
|
}, |
|
"bad_words_ids": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"bench_dataset": { |
|
"desc": null, |
|
"value": "pharaouk/dharma-1/dharma_1_mini.json" |
|
}, |
|
"do_bench_eval": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"jit_mode_eval": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"learning_rate": { |
|
"desc": null, |
|
"value": 0.0002 |
|
}, |
|
"logging_steps": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"max_grad_norm": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"mp_parameters": { |
|
"desc": null, |
|
"value": "" |
|
}, |
|
"output_scores": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"save_strategy": { |
|
"desc": null, |
|
"value": "steps" |
|
}, |
|
"split_batches": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"torch_compile": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"tpu_num_cores": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"attention_bias": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"bf16_full_eval": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"early_stopping": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"fp16_full_eval": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"fp16_opt_level": { |
|
"desc": null, |
|
"value": "O1" |
|
}, |
|
"length_penalty": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"max_seq_length": { |
|
"desc": null, |
|
"value": 4096 |
|
}, |
|
"sample_packing": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"tf_legacy_loss": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"use_mps_device": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"finetuning_task": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"group_by_length": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"hub_always_push": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"num_beam_groups": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"save_only_model": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"suppress_tokens": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"tokenizer_class": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"dispatch_batches": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"full_determinism": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"hub_private_repo": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"ignore_data_skip": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"log_on_each_node": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"logging_strategy": { |
|
"desc": null, |
|
"value": "steps" |
|
}, |
|
"num_train_epochs": { |
|
"desc": null, |
|
"value": 8 |
|
}, |
|
"save_safetensors": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"save_total_limit": { |
|
"desc": null, |
|
"value": 4 |
|
}, |
|
"attention_dropout": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"ddp_bucket_cap_mb": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"diversity_penalty": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"do_causal_lm_eval": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"greater_is_better": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"initializer_range": { |
|
"desc": null, |
|
"value": 0.02 |
|
}, |
|
"intermediate_size": { |
|
"desc": null, |
|
"value": 16384 |
|
}, |
|
"log_level_replica": { |
|
"desc": null, |
|
"value": "warning" |
|
}, |
|
"loraplus_lr_ratio": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"lr_scheduler_type": { |
|
"desc": null, |
|
"value": "cosine" |
|
}, |
|
"max_bench_samples": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"num_hidden_layers": { |
|
"desc": null, |
|
"value": 18 |
|
}, |
|
"output_attentions": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"push_to_hub_token": { |
|
"desc": null, |
|
"value": "<PUSH_TO_HUB_TOKEN>" |
|
}, |
|
"save_on_each_node": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"tpu_metrics_debug": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"accelerator_config": { |
|
"desc": null, |
|
"value": { |
|
"even_batches": true, |
|
"split_batches": false, |
|
"dispatch_batches": null, |
|
"use_seedable_sampler": true |
|
} |
|
}, |
|
"is_encoder_decoder": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"length_column_name": { |
|
"desc": null, |
|
"value": "length" |
|
}, |
|
"logging_first_step": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"relora_prune_ratio": { |
|
"desc": null, |
|
"value": 0.9 |
|
}, |
|
"repetition_penalty": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"torch_compile_mode": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"add_cross_attention": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"cosine_min_lr_ratio": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"eval_sample_packing": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"evaluation_strategy": { |
|
"desc": null, |
|
"value": "steps" |
|
}, |
|
"forced_bos_token_id": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"forced_eos_token_id": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"fsdp_min_num_params": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"lr_quadratic_warmup": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"lr_scheduler_kwargs": { |
|
"desc": null, |
|
"value": {} |
|
}, |
|
"neftune_noise_alpha": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"num_attention_heads": { |
|
"desc": null, |
|
"value": 8 |
|
}, |
|
"num_key_value_heads": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"quantization_config": { |
|
"desc": null, |
|
"value": { |
|
"load_in_4bit": true, |
|
"load_in_8bit": false, |
|
"quant_method": "QuantizationMethod.BITS_AND_BYTES", |
|
"_load_in_4bit": true, |
|
"_load_in_8bit": false, |
|
"llm_int8_threshold": 6, |
|
"bnb_4bit_quant_type": "nf4", |
|
"llm_int8_skip_modules": null, |
|
"bnb_4bit_compute_dtype": "bfloat16", |
|
"bnb_4bit_quant_storage": "uint8", |
|
"llm_int8_has_fp16_weight": false, |
|
"bnb_4bit_use_double_quant": true, |
|
"llm_int8_enable_fp32_cpu_offload": false |
|
} |
|
}, |
|
"relora_anneal_steps": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"relora_warmup_steps": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"skip_memory_metrics": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"tie_encoder_decoder": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"tie_word_embeddings": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"auto_find_batch_size": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"bench_source_max_len": { |
|
"desc": null, |
|
"value": 2048 |
|
}, |
|
"dataloader_drop_last": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"no_repeat_ngram_size": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"num_return_sequences": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"optim_target_modules": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"output_hidden_states": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"overwrite_output_dir": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"prediction_loss_only": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"push_to_hub_model_id": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"task_specific_params": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"transformers_version": { |
|
"desc": null, |
|
"value": "4.39.0.dev0" |
|
}, |
|
"begin_suppress_tokens": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"dataloader_pin_memory": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"ddp_broadcast_buffers": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"loraplus_lr_embedding": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"metric_for_best_model": { |
|
"desc": null, |
|
"value": "loss" |
|
}, |
|
"remove_invalid_values": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"remove_unused_columns": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"torch_compile_backend": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"dataloader_num_workers": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"decoder_start_token_id": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"gradient_checkpointing": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"half_precision_backend": { |
|
"desc": null, |
|
"value": "auto" |
|
}, |
|
"label_smoothing_factor": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"load_best_model_at_end": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"logging_nan_inf_filter": { |
|
"desc": null, |
|
"value": true |
|
}, |
|
"multipack_real_batches": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"resume_from_checkpoint": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"chunk_size_feed_forward": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"eval_accumulation_steps": { |
|
"desc": null, |
|
"value": 3 |
|
}, |
|
"max_position_embeddings": { |
|
"desc": null, |
|
"value": 8192 |
|
}, |
|
"per_gpu_eval_batch_size": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"return_dict_in_generate": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"cosine_constant_lr_ratio": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"per_gpu_train_batch_size": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"push_to_hub_organization": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"include_tokens_per_second": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"sample_packing_efficiency": { |
|
"desc": null, |
|
"value": 1 |
|
}, |
|
"dataloader_prefetch_factor": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"ddp_find_unused_parameters": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"include_inputs_for_metrics": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"per_device_eval_batch_size": { |
|
"desc": null, |
|
"value": 2 |
|
}, |
|
"use_legacy_prediction_loop": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"cross_attention_hidden_size": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"gradient_accumulation_steps": { |
|
"desc": null, |
|
"value": 3 |
|
}, |
|
"per_device_train_batch_size": { |
|
"desc": null, |
|
"value": 2 |
|
}, |
|
"encoder_no_repeat_ngram_size": { |
|
"desc": null, |
|
"value": 0 |
|
}, |
|
"dataloader_persistent_workers": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"gradient_checkpointing_kwargs": { |
|
"desc": null, |
|
"value": { |
|
"use_reentrant": true |
|
} |
|
}, |
|
"include_num_input_tokens_seen": { |
|
"desc": null, |
|
"value": false |
|
}, |
|
"exponential_decay_length_penalty": { |
|
"desc": null, |
|
"value": null |
|
}, |
|
"sample_packing_seq_len_multiplier": { |
|
"desc": null, |
|
"value": 2 |
|
}, |
|
"fsdp_transformer_layer_cls_to_wrap": { |
|
"desc": null, |
|
"value": null |
|
} |
|
} |
|
### Model Architecture and Objective |
|
|
|
## Citation [optional] |
|
|
|
|
|
## Glossary [optional] |
|
### Framework versions |
|
|
|
- PEFT 0.9.0 |
