File size: 4,382 Bytes
96fe658 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
kwargs = {
'per_device_train_batch_size': 2,
'per_device_eval_batch_size': 2,
'save_steps': 50,
'gradient_accumulation_steps': 1,
'num_train_epochs': 1,
}
SYSTEM_PROMPT = ('A conversation between User and Assistant. The user asks a question, and the Assistant solves it. '
'The assistant first thinks about the reasoning process in the mind and then provides the user '
'with the answer. The reasoning process and answer are enclosed within <think> </think> '
'and <answer> </answer> tags, respectively, i.e., <think> reasoning process here </think><answer> '
'answer here </answer>')
def test_llm():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='grpo',
model='Qwen/Qwen2.5-1.5B-Instruct',
train_type='full',
dataset=['AI-MO/NuminaMath-TIR#100'],
split_dataset_ratio=0.1,
system=SYSTEM_PROMPT,
reward_funcs=['accuracy', 'format'],
max_completion_length=4096,
num_generations=2,
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_llm_zero2():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='grpo',
model='Qwen/Qwen2.5-1.5B-Instruct',
train_type='full',
dataset=['AI-MO/NuminaMath-TIR#100'],
system=SYSTEM_PROMPT,
reward_funcs=['accuracy', 'format'],
max_completion_length=4096,
num_generations=2,
deepspeed='zero2',
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_llm_vllm():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='grpo',
model='Qwen/Qwen2.5-1.5B-Instruct',
reward_model='AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft',
train_type='full',
dataset=['AI-MO/NuminaMath-TIR#100'],
system=SYSTEM_PROMPT,
reward_funcs=['accuracy', 'format'],
use_vllm=True,
max_completion_length=4096,
num_generations=2,
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_llm_vllm_zero2():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='grpo',
model='Qwen/Qwen2.5-1.5B-Instruct',
train_type='full',
dataset=['AI-MO/NuminaMath-TIR#100'],
system=SYSTEM_PROMPT,
reward_funcs=['accuracy', 'format'],
use_vllm=True,
max_completion_length=4096,
num_generations=2,
deepspeed='zero2',
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_mllm_pt():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='grpo',
model='Qwen/Qwen2-VL-2B-Instruct',
train_type='full',
# dataset=['AI-MO/NuminaMath-TIR#100'],
dataset=['modelscope/coco_2014_caption:validation#100'],
system=SYSTEM_PROMPT,
reward_funcs=['format'],
max_completion_length=4096,
num_generations=2,
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
if __name__ == '__main__':
# test_llm()
# test_llm_zero3()
# test_llm_vllm()
# test_llm_vllm_zero2()
test_mllm_pt()
|