| from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig | |
| model_id = "elyza/ELYZA-japanese-Llama-2-13b-fast-instruct" | |
| tokenizer = AutoTokenizer.from_pretrained(model_id) | |
| gptq_config = GPTQConfig(bits=4, dataset = "c4", tokenizer=tokenizer, group_size=64) | |
| model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda", quantization_config=gptq_config) | |
| tokenizer.save_pretrained("./quantized") | |
| model.save_pretrained("./quantized") | |