Upload 3 files

adapter/adapter_config.json

@@ -10,22 +10,22 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
+  "lora_alpha": 16,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 8,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "o_proj",
     "up_proj",
-    "gate_proj",
     "k_proj",
+    "o_proj",
+    "down_proj",
     "q_proj",
+    "gate_proj",
     "v_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6e200a8e6d5127eeea2ebca01bcf1314b8312dc4bed280fcf3b19f4c12def93
-size 83946192
+oid sha256:141e7b3cfb0c6232028bb14bab844dc0b18de1d556f32015fc2f281f39561a72
+size 42002584

axolotl/training.yaml

@@ -11,7 +11,7 @@ datasets:
     type: alpaca
 dataset_prepared_path: last_run_prepared
 val_set_size: 0.01
-output_dir: ./out/finbro-v0.1.0-llama-3-8B-instruct-4bit-1m-POSE
+output_dir: ./out/finbro-v0.1.0-llama-3-8B-instruct-1m
 
 adapter: qlora
 lora_model_dir:
@@ -20,22 +20,23 @@ sequence_len: 8192
 sample_packing: false
 pad_to_sequence_len: true
 
-lora_r: 16
-lora_alpha: 32
+lora_r: 8
+lora_alpha: 16
 lora_dropout: 0.05
 lora_target_modules:
 lora_target_linear: true
 lora_fan_in_fan_out:
 
-wandb_project: finbro-v0.1.0-llama-3-8B-instruct-4bit-1m-POSE
+wandb_project: finbro-v0.1.0-llama-3-8B-instruct-131k
 wandb_entity: sigmance
 wandb_watch: "true"
-wandb_name: finbro-v0.1.0-llama-3-8B-instruct-4bit-1m-POSE
+wandb_name: finbro-v0.1.0-llama-3-8B-instruct-1m
 wandb_log_model: "true"
 
 use_pose: true
 pose_max_context_len: 1048576
 
+# lora_on_cpu: 
 overrides_of_model_config:
   rope_theta: 500000.0
   max_position_embeddings: 1048576
@@ -58,7 +59,7 @@ gradient_checkpointing: true
 gradient_checkpointing_kwargs:
   use_reentrant: true
 early_stopping_patience: 50
-resume_from_checkpoint: ./out/finbro-v0.1.0-llama-3-8B-instruct-4bit-1m-POSE/checkpoint-4800
+resume_from_checkpoint: ./out/finbro-v0.1.0-llama-3-8B-instruct-131k/checkpoint-3500
 local_rank:
 logging_steps: 1
 xformers_attention:
@@ -72,7 +73,7 @@ eval_steps: 100
 eval_table_size:
 # saves_per_epoch: 1
 debug:
-deepspeed:
+deepspeed: 
 weight_decay: 0.0
 fsdp:
   - full_shard