Add config
Browse files
clone_sentdex_model_tokenizer.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
| 1 |
#!/usr/bin/env python
|
| 2 |
-
from transformers import AutoTokenizer, TFAutoModelForCausalLM, AutoModelForCausalLM
|
| 3 |
import tensorflow as tf
|
| 4 |
|
|
|
|
| 5 |
tokenizer = AutoTokenizer.from_pretrained("Sentdex/GPyT")
|
| 6 |
tf_model = TFAutoModelForCausalLM.from_pretrained("Sentdex/GPyT")
|
| 7 |
pytorch_model = AutoModelForCausalLM.from_pretrained("Sentdex/GPyT")
|
|
|
|
|
|
|
| 8 |
tokenizer.save_pretrained(save_directory='./')
|
| 9 |
tf_model.save_pretrained(save_directory='./', saved_model=True)
|
| 10 |
pytorch_model.save_pretrained(save_directory='./')
|
|
|
|
| 1 |
#!/usr/bin/env python
|
| 2 |
+
from transformers import AutoTokenizer, TFAutoModelForCausalLM, AutoModelForCausalLM, GPT2Config
|
| 3 |
import tensorflow as tf
|
| 4 |
|
| 5 |
+
config = GPT2Config.from_pretrained("Sentdex/GPyT")
|
| 6 |
tokenizer = AutoTokenizer.from_pretrained("Sentdex/GPyT")
|
| 7 |
tf_model = TFAutoModelForCausalLM.from_pretrained("Sentdex/GPyT")
|
| 8 |
pytorch_model = AutoModelForCausalLM.from_pretrained("Sentdex/GPyT")
|
| 9 |
+
|
| 10 |
+
config.save_pretrained("./")
|
| 11 |
tokenizer.save_pretrained(save_directory='./')
|
| 12 |
tf_model.save_pretrained(save_directory='./', saved_model=True)
|
| 13 |
pytorch_model.save_pretrained(save_directory='./')
|