File size: 2,297 Bytes
86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d 86b80ce 20df87d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
#!/usr/bin/env python3
import os
from transformers import AutoTokenizer, GPT2Tokenizer
#from megatron.initialize import initialize_megatron
from metaseq import checkpoint_utils
from transformers import OPTForCausalLM
import torch
path = "./model"
hf_path = "/home/patrick/facebook/opt-2.7b"
vocab_file = os.path.join(path, "gpt2-vocab.json")
merges_file = os.path.join(path, "gpt2-merges.txt")
tokenizer = GPT2Tokenizer(vocab_file, merges_file)
tokenizer.save_pretrained(path)
checkpoint = checkpoint_utils.load_model_ensemble_and_task(
[os.path.join(path, "restored.pt")],
arg_overrides={
"vocab_filename": vocab_file,
"merges_filename": merges_file,
}
)
model = checkpoint[0][0].eval()
model = model
hf_model = OPTForCausalLM.from_pretrained(hf_path)
# forward passes
def single_batch_forward_logits(prompts):
input_ids = tokenizer(prompts, return_tensors="pt").input_ids
input_ids = torch.cat([torch.tensor([[0]]), input_ids], dim=-1)
input_ids = input_ids
with torch.no_grad():
logits = model(input_ids)[0]
return logits
# forward hf
def forward_hf(prompts):
input_ids = tokenizer(prompts, return_tensors="pt").input_ids
input_ids = torch.cat([torch.tensor([[0]]), input_ids], dim=-1)
input_ids = input_ids
with torch.no_grad():
logits = hf_model(input_ids)[0]
return logits
prompts = [
"Today is a beautiful day and I want to",
"In the city of",
"Paris is the capital of France and",
"Computers and mobile phones have taken",
]
print("Next word generation")
for prompt in prompts:
print("-------------")
print(f"Prompt: {prompt}...\n")
logits_fsq = single_batch_forward_logits(prompt)
pred_next_token = torch.argmax(logits_fsq[0, -1], -1)
next_token = tokenizer.convert_ids_to_tokens([pred_next_token])
next_token = next_token[0].replace("Ġ", "")
print(f"Next word: {next_token}")
print("-------------")
logits = forward_hf(prompt)
pred_next_token = torch.argmax(logits[0, -1], -1)
next_token = tokenizer.convert_ids_to_tokens([pred_next_token])
next_token = next_token[0].replace("Ġ", "")
print(f"Next word: {next_token}")
print("-------------")
print("Is equal:", torch.allclose(logits_fsq.cpu(), logits.cpu(), atol=1e-3))
|