Training in progress, step 100

adapter_config.json

@@ -10,22 +10,22 @@
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 128,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 256,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "q_proj",
-    "up_proj",
     "gate_proj",
+    "q_proj",
     "o_proj",
-    "down_proj",
-    "v_proj"
+    "v_proj",
+    "up_proj",
+    "k_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:728f17232ede782bd4d8adfed009764b81f26ca9aa2ecc7aec3b6836f207e997
-size 2558587064
+oid sha256:8d521e28dbcddbaf7d4d927823f1737079371d39ca4f7efa1d0e600344b66683
+size 639691872

angle.config

@@ -5,18 +5,18 @@
   "pooling_strategy": "cls",
   "lora_config_kwargs": {
     "task_type": "CAUSAL_LM",
-    "r": 256,
+    "r": 64,
     "lora_alpha": 128,
-    "lora_dropout": 0.1,
+    "lora_dropout": 0.05,
     "bias": "none",
     "target_modules": [
-      "q_proj",
-      "v_proj",
       "k_proj",
-      "down_proj",
       "o_proj",
+      "q_proj",
+      "down_proj",
+      "gate_proj",
       "up_proj",
-      "gate_proj"
+      "v_proj"
     ]
   },
   "apply_lora": 1

runs/Apr07_08-07-05_instance-20240226-074643/events.out.tfevents.1712477231.instance-20240226-074643.980655.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fb7c6900beec7446589ce5d4ab16d668ad97198d604d9c3301434f0baa6bc1f
+size 5486

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce3d0279f2096d91e3d0b0a1c48adfb48125a2c4ce69ac510eb02420fb32d255
+oid sha256:f8ac4d46d101851b8d20dea76a03f66454781102688a534e4d697f8a77cfdfb8
 size 5048