Harsh Trivedi
commited on
Commit
•
8e97603
1
Parent(s):
a161ee5
update.
Browse files
README.md
CHANGED
@@ -30,7 +30,7 @@ Please checkout the details in our [github repository](https://github.com/stonyb
|
|
30 |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
31 |
from digit_tokenization import enable_digit_tokenization # digit_tokenization.py from https://github.com/stonybrooknlp/teabreac
|
32 |
|
33 |
-
model_name = "poet-large-iirc-retrieved"
|
34 |
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) # Fast doesn't work with digit tokenization
|
35 |
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
|
36 |
enable_digit_tokenization(tokenizer)
|
@@ -43,7 +43,7 @@ input_ids = tokenizer(
|
|
43 |
input_texts, return_tensors="pt",
|
44 |
truncation=True, max_length=800,
|
45 |
add_special_tokens=True, padding=True,
|
46 |
-
)
|
47 |
generated_ids = model.generate(input_ids, min_length=1, max_length=50)
|
48 |
generated_predictions = tokenizer.batch_decode(generated_ids, skip_special_tokens=False)
|
49 |
generated_predictions = [
|
|
|
30 |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
31 |
from digit_tokenization import enable_digit_tokenization # digit_tokenization.py from https://github.com/stonybrooknlp/teabreac
|
32 |
|
33 |
+
model_name = "StonyBrookNLP/poet-large-iirc-retrieved"
|
34 |
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) # Fast doesn't work with digit tokenization
|
35 |
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
|
36 |
enable_digit_tokenization(tokenizer)
|
|
|
43 |
input_texts, return_tensors="pt",
|
44 |
truncation=True, max_length=800,
|
45 |
add_special_tokens=True, padding=True,
|
46 |
+
)["input_ids"]
|
47 |
generated_ids = model.generate(input_ids, min_length=1, max_length=50)
|
48 |
generated_predictions = tokenizer.batch_decode(generated_ids, skip_special_tokens=False)
|
49 |
generated_predictions = [
|