Commit
•
d6f82aa
1
Parent(s):
2b9d925
Upload README.md
Browse files
README.md
CHANGED
@@ -16,7 +16,7 @@ pipeline_tag: text-generation
|
|
16 |
---
|
17 |
|
18 |
|
19 |
-
# LlaMa 2 7b 4-bit Python Coder 👩💻
|
20 |
|
21 |
**LlaMa-2 7b** fine-tuned on the **CodeAlpaca 20k instructions dataset** by using the method **QLoRA** in 4-bit with [PEFT](https://github.com/huggingface/peft) library.
|
22 |
|
@@ -76,6 +76,30 @@ The following `bitsandbytes` quantization config was used during training:
|
|
76 |
### Framework versions
|
77 |
- PEFT 0.4.0
|
78 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
79 |
### Example of usage
|
80 |
```py
|
81 |
import torch
|
@@ -117,10 +141,10 @@ print(f"Generated instruction:\n{tokenizer.batch_decode(outputs.detach().cpu().n
|
|
117 |
```
|
118 |
@misc {edumunozsala_2023,
|
119 |
author = { {Eduardo Muñoz} },
|
120 |
-
title = { llama-2-7b-int4-python-coder
|
121 |
year = 2023,
|
122 |
url = { https://huggingface.co/edumunozsala/llama-2-7b-int4-python-18k-alpaca },
|
123 |
-
doi = {
|
124 |
publisher = { Hugging Face }
|
125 |
}
|
126 |
```
|
|
|
16 |
---
|
17 |
|
18 |
|
19 |
+
# LlaMa 2 7b 4-bit Python Coder 👩💻 :man_technologist:
|
20 |
|
21 |
**LlaMa-2 7b** fine-tuned on the **CodeAlpaca 20k instructions dataset** by using the method **QLoRA** in 4-bit with [PEFT](https://github.com/huggingface/peft) library.
|
22 |
|
|
|
76 |
### Framework versions
|
77 |
- PEFT 0.4.0
|
78 |
|
79 |
+
### Training metrics
|
80 |
+
{'loss': 1.044, 'learning_rate': 3.571428571428572e-05, 'epoch': 0.01}
|
81 |
+
{'loss': 0.8413, 'learning_rate': 7.142857142857143e-05, 'epoch': 0.01}
|
82 |
+
{'loss': 0.7299, 'learning_rate': 0.00010714285714285715, 'epoch': 0.02}
|
83 |
+
{'loss': 0.6593, 'learning_rate': 0.00014285714285714287, 'epoch': 0.02}
|
84 |
+
{'loss': 0.6309, 'learning_rate': 0.0001785714285714286, 'epoch': 0.03}
|
85 |
+
{'loss': 0.5916, 'learning_rate': 0.00019999757708974043, 'epoch': 0.03}
|
86 |
+
{'loss': 0.5861, 'learning_rate': 0.00019997032069768138, 'epoch': 0.04}
|
87 |
+
{'loss': 0.6118, 'learning_rate': 0.0001999127875580558, 'epoch': 0.04}
|
88 |
+
{'loss': 0.5928, 'learning_rate': 0.00019982499509519857, 'epoch': 0.05}
|
89 |
+
{'loss': 0.5978, 'learning_rate': 0.00019970696989770335, 'epoch': 0.05}
|
90 |
+
{'loss': 0.5791, 'learning_rate': 0.0001995587477103701, 'epoch': 0.06}
|
91 |
+
{'loss': 0.6054, 'learning_rate': 0.00019938037342337933, 'epoch': 0.06}
|
92 |
+
{'loss': 0.5864, 'learning_rate': 0.00019917190105869708, 'epoch': 0.07}
|
93 |
+
{'loss': 0.6159, 'learning_rate': 0.0001989333937537136, 'epoch': 0.08}
|
94 |
+
{'loss': 0.583, 'learning_rate': 0.00019866492374212205, 'epoch': 0.08}
|
95 |
+
{'loss': 0.6066, 'learning_rate': 0.00019836657233204182, 'epoch': 0.09}
|
96 |
+
{'loss': 0.5934, 'learning_rate': 0.00019803842988139374, 'epoch': 0.09}
|
97 |
+
{'loss': 0.5836, 'learning_rate': 0.00019768059577053473, 'epoch': 0.1}
|
98 |
+
{'loss': 0.6021, 'learning_rate': 0.00019729317837215943, 'epoch': 0.1}
|
99 |
+
{'loss': 0.5659, 'learning_rate': 0.00019687629501847898, 'epoch': 0.11}
|
100 |
+
{'loss': 0.5754, 'learning_rate': 0.00019643007196568606, 'epoch': 0.11}
|
101 |
+
{'loss': 0.5936, 'learning_rate': 0.000195954644355717, 'epoch': 0.12}
|
102 |
+
|
103 |
### Example of usage
|
104 |
```py
|
105 |
import torch
|
|
|
141 |
```
|
142 |
@misc {edumunozsala_2023,
|
143 |
author = { {Eduardo Muñoz} },
|
144 |
+
title = { llama-2-7b-int4-python-coder },
|
145 |
year = 2023,
|
146 |
url = { https://huggingface.co/edumunozsala/llama-2-7b-int4-python-18k-alpaca },
|
147 |
+
doi = { },
|
148 |
publisher = { Hugging Face }
|
149 |
}
|
150 |
```
|