msr2000
commited on
Commit
•
ca159e6
1
Parent(s):
d734100
Update README.md
Browse files
README.md
CHANGED
@@ -189,7 +189,7 @@ model_name = "deepseek-ai/DeepSeek-V2"
|
|
189 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
190 |
# `max_memory` should be set based on your devices
|
191 |
max_memory = {i: "75GB" for i in range(8)}
|
192 |
-
model = AutoModelForCausalLM.from_pretrained(
|
193 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
194 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
195 |
|
@@ -210,7 +210,7 @@ model_name = "deepseek-ai/DeepSeek-V2-Chat"
|
|
210 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
211 |
# `max_memory` should be set based on your devices
|
212 |
max_memory = {i: "75GB" for i in range(8)}
|
213 |
-
model = AutoModelForCausalLM.from_pretrained(
|
214 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
215 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
216 |
|
|
|
189 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
190 |
# `max_memory` should be set based on your devices
|
191 |
max_memory = {i: "75GB" for i in range(8)}
|
192 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory)
|
193 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
194 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
195 |
|
|
|
210 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
211 |
# `max_memory` should be set based on your devices
|
212 |
max_memory = {i: "75GB" for i in range(8)}
|
213 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory)
|
214 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
215 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
216 |
|