styalai commited on
Commit
74b4047
1 Parent(s): d745422

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +114 -18
README.md CHANGED
@@ -4,12 +4,15 @@ tags:
4
  - text-generation-inference
5
  - text-generation
6
  - peft
 
7
  library_name: transformers
8
  widget:
9
- - messages:
10
- - role: user
11
- content: What is your favorite condiment?
12
  license: other
 
 
13
  ---
14
 
15
  # Model Trained Using AutoTrain
@@ -20,26 +23,119 @@ This model was trained using AutoTrain. For more information, please visit [Auto
20
 
21
  ```python
22
 
23
- from transformers import AutoModelForCausalLM, AutoTokenizer
 
24
 
25
- model_path = "PATH_TO_THIS_REPO"
26
 
27
- tokenizer = AutoTokenizer.from_pretrained(model_path)
28
  model = AutoModelForCausalLM.from_pretrained(
29
- model_path,
30
- device_map="auto",
31
- torch_dtype='auto'
32
- ).eval()
 
 
33
 
34
- # Prompt content: "hi"
35
  messages = [
36
- {"role": "user", "content": "hi"}
37
  ]
38
 
39
- input_ids = tokenizer.apply_chat_template(conversation=messages, tokenize=True, add_generation_prompt=True, return_tensors='pt')
40
- output_ids = model.generate(input_ids.to('cuda'))
41
- response = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)
 
 
42
 
43
- # Model response: "Hello! How can I assist you today?"
44
- print(response)
45
- ```
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  - text-generation-inference
5
  - text-generation
6
  - peft
7
+ - Phi 3
8
  library_name: transformers
9
  widget:
10
+ - messages:
11
+ - role: user
12
+ content: What is your favorite condiment?
13
  license: other
14
+ language:
15
+ - en
16
  ---
17
 
18
  # Model Trained Using AutoTrain
 
23
 
24
  ```python
25
 
26
+ import torch
27
+ from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
28
 
29
+ torch.random.manual_seed(0)
30
 
 
31
  model = AutoModelForCausalLM.from_pretrained(
32
+ "styalai/competition-math-phinetune-v1", q
33
+ device_map="cuda",
34
+ torch_dtype="auto",
35
+ trust_remote_code=True,
36
+ )
37
+ tokenizer = AutoTokenizer.from_pretrained("styalai/competition-math-phinetune-v1")
38
 
 
39
  messages = [
40
+ {"role": "user", "content": "What about solving an 2x + 3 = 7 equation?"},
41
  ]
42
 
43
+ pipe = pipeline(
44
+ "text-generation",
45
+ model=model,
46
+ tokenizer=tokenizer,
47
+ )
48
 
49
+ generation_args = {
50
+ "max_new_tokens": 500,
51
+ "return_full_text": False,
52
+ "temperature": 0.0,
53
+ "do_sample": False,
54
+ }
55
+
56
+ output = pipe(messages, **generation_args)
57
+ print(output[0]['generated_text'])
58
+ ```
59
+
60
+ # Info
61
+
62
+ Fine-tune from styalai/phi-ne-tuning-1-4 who it fine tune from phi-3
63
+
64
+ parameters of autotrain :
65
+ ```python
66
+ project_name = 'competition-math-phinetune-v1' # @param {type:"string"}
67
+ model_name = "styalai/phi-ne-tuning-1-4" #'microsoft/Phi-3-mini-4k-instruct' # @param {type:"string"}
68
+
69
+ #@markdown ---
70
+ #@markdown #### Push to Hub?
71
+ #@markdown Use these only if you want to push your trained model to a private repo in your Hugging Face Account
72
+ #@markdown If you dont use these, the model will be saved in Google Colab and you are required to download it manually.
73
+ #@markdown Please enter your Hugging Face write token. The trained model will be saved to your Hugging Face account.
74
+ #@markdown You can find your token here: https://huggingface.co/settings/tokens
75
+ push_to_hub = True # @param ["False", "True"] {type:"raw"}
76
+ hf_token = "hf_****" #@param {type:"string"}
77
+ #repo_id = "styalai/phine_tuning_1" #@param {type:"string"}
78
+
79
+ #@markdown ---
80
+ #@markdown #### Hyperparameters
81
+ learning_rate = 3e-4 # @param {type:"number"}
82
+ num_epochs = 1 #@param {type:"number"}
83
+ batch_size = 1 # @param {type:"slider", min:1, max:32, step:1}
84
+ block_size = 1024 # @param {type:"number"}
85
+ trainer = "sft" # @param ["default", "sft"] {type:"raw"}
86
+ warmup_ratio = 0.1 # @param {type:"number"}
87
+ weight_decay = 0.01 # @param {type:"number"}
88
+ gradient_accumulation = 4 # @param {type:"number"}
89
+ mixed_precision = "fp16" # @param ["fp16", "bf16", "none"] {type:"raw"}
90
+ peft = True # @param ["False", "True"] {type:"raw"}
91
+ quantization = "int4" # @param ["int4", "int8", "none"] {type:"raw"}
92
+ lora_r = 16 #@param {type:"number"}
93
+ lora_alpha = 32 #@param {type:"number"}
94
+ lora_dropout = 0.05 #@param {type:"number"}
95
+
96
+ code for the creation of the dataset :
97
+ from datasets import load_dataset
98
+ dataset = load_dataset("camel-ai/math")#, streaming=True)
99
+
100
+ import pandas as pd
101
+ data = {"text":[]}
102
+
103
+ msg1 = dataset["train"]["message_1"]
104
+ msg2 = dataset["train"]["message_2"]
105
+
106
+ for i in range(3500):
107
+ user = "<|user|>"+ msg1[i] +"<|end|>\n"
108
+ phi = "<|assistant|>"+ msg2[i] +"<|end|>"
109
+ prompt = user+phi
110
+ data["text"].append(prompt)
111
+
112
+ data = pd.DataFrame.from_dict(data)
113
+ print(data)
114
+ #os.mkdir("/kaggle/working/data")
115
+ data.to_csv('data/dataset.csv', index=False, escapechar='\\')
116
+
117
+ !autotrain llm \
118
+ --train \
119
+ --username "styalai" \
120
+ --merge-adapter \
121
+ --model ${MODEL_NAME} \
122
+ --project-name ${PROJECT_NAME} \
123
+ --data-path data/ \
124
+ --text-column text \
125
+ --lr ${LEARNING_RATE} \
126
+ --batch-size ${BATCH_SIZE} \
127
+ --epochs ${NUM_EPOCHS} \
128
+ --block-size ${BLOCK_SIZE} \
129
+ --warmup-ratio ${WARMUP_RATIO} \
130
+ --lora-r ${LORA_R} \
131
+ --lora-alpha ${LORA_ALPHA} \
132
+ --lora-dropout ${LORA_DROPOUT} \
133
+ --weight-decay ${WEIGHT_DECAY} \
134
+ --gradient-accumulation ${GRADIENT_ACCUMULATION} \
135
+ --quantization ${QUANTIZATION} \
136
+ --mixed-precision ${MIXED_PRECISION} \
137
+ $( [[ "$PEFT" == "True" ]] && echo "--peft" ) \
138
+ $( [[ "$PUSH_TO_HUB" == "True" ]] && echo "--push-to-hub --token ${HF_TOKEN}" )q
139
+ ```
140
+
141
+ durée de l’entrainement : 1:07:41