File size: 2,615 Bytes
96fe658 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
kwargs = {
'per_device_train_batch_size': 2,
'save_steps': 5,
'gradient_accumulation_steps': 4,
'num_train_epochs': 1,
}
def test_rm():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='rm',
model='Shanghai_AI_Laboratory/internlm2-1_8b-reward',
dataset=['hjh0119/shareAI-Llama3-DPO-zh-en-emoji#100'],
split_dataset_ratio=0.01,
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_ppo():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='ppo',
model='LLM-Research/Llama-3.2-1B-Instruct',
reward_model='AI-ModelScope/GRM-Llama3.2-3B-rewardmodel-ft',
dataset=['AI-ModelScope/alpaca-gpt4-data-zh#100', 'AI-ModelScope/alpaca-gpt4-data-en#100'],
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_ppo2():
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='ppo',
model='Qwen/Qwen2.5-7B-Instruct',
reward_model='Qwen/Qwen2.5-7B-Instruct',
dataset=['AI-ModelScope/alpaca-gpt4-data-zh#100', 'AI-ModelScope/alpaca-gpt4-data-en#100'],
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
def test_ppo_vl():
# PPO currently does not support VL, and the image has not been uploaded.
os.environ['MAX_PIXELS'] = '1003520'
from swift.llm import rlhf_main, RLHFArguments, infer_main, InferArguments
result = rlhf_main(
RLHFArguments(
rlhf_type='ppo',
model='Qwen/Qwen2-VL-2B-Instruct',
reward_model='Qwen/Qwen2-VL-7B-Instruct',
dataset=['modelscope/coco_2014_caption:validation#100'],
**kwargs))
last_model_checkpoint = result['last_model_checkpoint']
infer_main(InferArguments(adapters=last_model_checkpoint, load_data_args=True, merge_lora=True))
if __name__ == '__main__':
# test_rm()
# test_ppo()
# test_ppo2()
test_ppo_vl()
|