upload files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+adapter_model.bin filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,31 @@
 ---
 license: mit
+datasets:
+- yahma/alpaca-cleaned
 ---
+
+This repo contains a low-rank adapter for LLaMA-7b
+fit on the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset.
+
+This version of the weights was trained with the following hyperparameters:
+
+- Epochs: 10 (load from best epoch)
+- Batch size: 128
+- Cutoff length: 512
+- Learning rate: 3e-4
+- Lora _r_: 16
+- Lora target modules: q_proj, k_proj, v_proj, o_proj
+
+That is:
+
+```
+python finetune.py \
+    --base_model='baffo32/decapoda-research-llama-7B-hf' \
+    --num_epochs=10 \
+    --cutoff_len=512 \
+    --group_by_length \
+    --output_dir='./lora-alpaca-512-qkvo' \
+    --lora_target_modules='[q_proj,k_proj,v_proj,o_proj]' \
+    --lora_r=16 \
+    --micro_batch_size=8
+```

adapter_config.json

@@ -0,0 +1,19 @@
+{
+  "base_model_name_or_path": "baffo32/decapoda-research-llama-7B-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "merge_weights": false,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e7187f51fbdeff8815046d30f0a325e43491040e6eac8cec5e2ba64f1e87807
+size 67201357