Update README.md
Browse files
README.md
CHANGED
@@ -16,30 +16,127 @@ license: other
|
|
16 |
|
17 |
This model was trained using AutoTrain. For more information, please visit [AutoTrain](https://hf.co/docs/autotrain).
|
18 |
|
|
|
|
|
|
|
|
|
19 |
# Usage
|
20 |
|
21 |
```python
|
22 |
|
23 |
-
|
|
|
24 |
|
25 |
-
|
26 |
|
27 |
-
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
28 |
model = AutoModelForCausalLM.from_pretrained(
|
29 |
-
|
30 |
-
device_map="
|
31 |
-
torch_dtype=
|
32 |
-
|
|
|
|
|
33 |
|
34 |
-
# Prompt content: "hi"
|
35 |
messages = [
|
36 |
-
{"role": "user", "content": "
|
37 |
]
|
38 |
|
39 |
-
|
40 |
-
|
41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
42 |
|
43 |
-
|
44 |
-
print(response)
|
45 |
-
```
|
|
|
16 |
|
17 |
This model was trained using AutoTrain. For more information, please visit [AutoTrain](https://hf.co/docs/autotrain).
|
18 |
|
19 |
+
# Model Trained Using AutoTrain
|
20 |
+
|
21 |
+
This model was trained using AutoTrain. For more information, please visit [AutoTrain](https://hf.co/docs/autotrain).
|
22 |
+
|
23 |
# Usage
|
24 |
|
25 |
```python
|
26 |
|
27 |
+
import torch
|
28 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
|
29 |
|
30 |
+
torch.random.manual_seed(0)
|
31 |
|
|
|
32 |
model = AutoModelForCausalLM.from_pretrained(
|
33 |
+
"styalai/competition-math-phinetune-v1", q
|
34 |
+
device_map="cuda",
|
35 |
+
torch_dtype="auto",
|
36 |
+
trust_remote_code=True,
|
37 |
+
)
|
38 |
+
tokenizer = AutoTokenizer.from_pretrained("styalai/competition-math-phinetune-v1")
|
39 |
|
|
|
40 |
messages = [
|
41 |
+
{"role": "user", "content": "What about solving an 2x + 3 = 7 equation?"},
|
42 |
]
|
43 |
|
44 |
+
pipe = pipeline(
|
45 |
+
"text-generation",
|
46 |
+
model=model,
|
47 |
+
tokenizer=tokenizer,
|
48 |
+
)
|
49 |
+
|
50 |
+
generation_args = {
|
51 |
+
"max_new_tokens": 500,
|
52 |
+
"return_full_text": False,
|
53 |
+
"temperature": 0.0,
|
54 |
+
"do_sample": False,
|
55 |
+
}
|
56 |
+
|
57 |
+
output = pipe(messages, **generation_args)
|
58 |
+
print(output[0]['generated_text'])
|
59 |
+
```
|
60 |
+
|
61 |
+
# Info
|
62 |
+
|
63 |
+
Fine-tune from styalai/phi-ne-tuning-1-4 who it fine tune from phi-3
|
64 |
+
|
65 |
+
parameters of autotrain :
|
66 |
+
```python
|
67 |
+
project_name = 'competition-math-phinetune-v1-1' # @param {type:"string"}
|
68 |
+
model_name = "styalai/competition-math-phinetune-v1" #'microsoft/Phi-3-mini-4k-instruct' # @param {type:"string"}
|
69 |
+
|
70 |
+
#@markdown ---
|
71 |
+
#@markdown #### Push to Hub?
|
72 |
+
#@markdown Use these only if you want to push your trained model to a private repo in your Hugging Face Account
|
73 |
+
#@markdown If you dont use these, the model will be saved in Google Colab and you are required to download it manually.
|
74 |
+
#@markdown Please enter your Hugging Face write token. The trained model will be saved to your Hugging Face account.
|
75 |
+
#@markdown You can find your token here: https://huggingface.co/settings/tokens
|
76 |
+
push_to_hub = True # @param ["False", "True"] {type:"raw"}
|
77 |
+
hf_token = "hf_****" #@param {type:"string"}
|
78 |
+
|
79 |
+
|
80 |
+
#@markdown ---
|
81 |
+
#@markdown #### Hyperparameters
|
82 |
+
learning_rate = 3e-4 # @param {type:"number"}
|
83 |
+
num_epochs = 1 #@param {type:"number"}
|
84 |
+
batch_size = 1 # @param {type:"slider", min:1, max:32, step:1}
|
85 |
+
block_size = 1024 # @param {type:"number"}
|
86 |
+
trainer = "sft" # @param ["default", "sft"] {type:"raw"}
|
87 |
+
warmup_ratio = 0.1 # @param {type:"number"}
|
88 |
+
weight_decay = 0.01 # @param {type:"number"}
|
89 |
+
gradient_accumulation = 4 # @param {type:"number"}
|
90 |
+
mixed_precision = "fp16" # @param ["fp16", "bf16", "none"] {type:"raw"}
|
91 |
+
peft = True # @param ["False", "True"] {type:"raw"}
|
92 |
+
quantization = "int4" # @param ["int4", "int8", "none"] {type:"raw"}
|
93 |
+
lora_r = 16 #@param {type:"number"}
|
94 |
+
lora_alpha = 32 #@param {type:"number"}
|
95 |
+
lora_dropout = 0.05 #@param {type:"number"}
|
96 |
+
|
97 |
+
code for the creation of the dataset :
|
98 |
+
from datasets import load_dataset
|
99 |
+
dataset = load_dataset("camel-ai/math")#, streaming=True)
|
100 |
+
|
101 |
+
import pandas as pd
|
102 |
+
data = {"text":[]}
|
103 |
+
|
104 |
+
msg1 = dataset["train"]["message_1"]
|
105 |
+
msg2 = dataset["train"]["message_2"]
|
106 |
+
|
107 |
+
for i in range(3500, 7000):
|
108 |
+
user = "<|user|>"+ msg1[i] +"<|end|>\n"
|
109 |
+
phi = "<|assistant|>"+ msg2[i] +"<|end|>"
|
110 |
+
prompt = user+phi
|
111 |
+
data["text"].append(prompt)
|
112 |
+
|
113 |
+
data = pd.DataFrame.from_dict(data)
|
114 |
+
print(data)
|
115 |
+
#os.mkdir("/kaggle/working/data")
|
116 |
+
data.to_csv('data/dataset.csv', index=False, escapechar='\\')
|
117 |
+
|
118 |
+
!autotrain llm \
|
119 |
+
--train \
|
120 |
+
--username "styalai" \
|
121 |
+
--merge-adapter \
|
122 |
+
--model ${MODEL_NAME} \
|
123 |
+
--project-name ${PROJECT_NAME} \
|
124 |
+
--data-path data/ \
|
125 |
+
--text-column text \
|
126 |
+
--lr ${LEARNING_RATE} \
|
127 |
+
--batch-size ${BATCH_SIZE} \
|
128 |
+
--epochs ${NUM_EPOCHS} \
|
129 |
+
--block-size ${BLOCK_SIZE} \
|
130 |
+
--warmup-ratio ${WARMUP_RATIO} \
|
131 |
+
--lora-r ${LORA_R} \
|
132 |
+
--lora-alpha ${LORA_ALPHA} \
|
133 |
+
--lora-dropout ${LORA_DROPOUT} \
|
134 |
+
--weight-decay ${WEIGHT_DECAY} \
|
135 |
+
--gradient-accumulation ${GRADIENT_ACCUMULATION} \
|
136 |
+
--quantization ${QUANTIZATION} \
|
137 |
+
--mixed-precision ${MIXED_PRECISION} \
|
138 |
+
$( [[ "$PEFT" == "True" ]] && echo "--peft" ) \
|
139 |
+
$( [[ "$PUSH_TO_HUB" == "True" ]] && echo "--push-to-hub --token ${HF_TOKEN}" )q
|
140 |
+
```
|
141 |
|
142 |
+
durée de l’entrainement : 1:07:41
|
|
|
|