Upload 3 files

adapter_config.json

@@ -0,0 +1,17 @@
+{
+    "bias": "none",
+    "peft_type": "MIXLORA",
+    "task_type": "CAUSAL_LM",
+    "r": 8,
+    "lora_alpha": 16,
+    "lora_dropout": 0.05,
+    "target_modules": [
+        "w1_proj",
+        "w2_proj",
+        "w3_proj"
+    ],
+    "routing_strategy": "mixtral",
+    "num_experts": 8,
+    "act_fn": "silu",
+    "top_k": 3
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ad12b855eddc19cb40331ea5d031130f690f7078a6bdec829bd1e5e98d73596
+size 375936965

mlora-train.json

@@ -0,0 +1,39 @@
+{
+    "cutoff_len": 1024,
+    "save_step": 2000,
+    "early_stop_test_step": 2000,
+    "train_lora_candidate_num": 1,
+    "train_lora_simultaneously_num": 1,
+    "train_strategy": "optim",
+    "lora": [
+        {
+            "name": "alpaca-mixlora-7b",
+            "optim": "adamw",
+            "lr": 3e-4,
+            "batch_size": 16,
+            "micro_batch_size": 4,
+            "test_batch_size": 64,
+            "num_epochs": 2,
+            "r": 8,
+            "lora_alpha": 16,
+            "lora_dropout": 0.05,
+            "target_modules": {
+                "q_proj": false,
+                "k_proj": false,
+                "v_proj": false,
+                "o_proj": false,
+                "w1_proj": true,
+                "w2_proj": true,
+                "w3_proj": true
+            },
+            "routing_strategy": "mixtral",
+            "num_experts": 8,
+            "top_k": 3,
+            "act_fn": "silu",
+            "data": "yahma/alpaca-cleaned",
+            "prompt": "template/alpaca.json",
+            "group_by_length": false,
+            "expand_side": "right"
+        }
+    ]
+}