Fix config usage
Browse files
clone_sentdex_model_tokenizer.py
CHANGED
@@ -2,12 +2,19 @@
|
|
2 |
from transformers import AutoTokenizer, TFAutoModelForCausalLM, AutoModelForCausalLM, GPT2Config
|
3 |
import tensorflow as tf
|
4 |
|
5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6 |
tokenizer = AutoTokenizer.from_pretrained("Sentdex/GPyT")
|
7 |
-
tf_model = TFAutoModelForCausalLM.from_pretrained("Sentdex/GPyT")
|
8 |
-
pytorch_model = AutoModelForCausalLM.from_pretrained("Sentdex/GPyT")
|
9 |
|
10 |
config.save_pretrained('./')
|
11 |
tokenizer.save_pretrained(save_directory='./')
|
12 |
-
tf_model.save_pretrained(save_directory='./', saved_model=True)
|
13 |
pytorch_model.save_pretrained(save_directory='./')
|
|
|
2 |
from transformers import AutoTokenizer, TFAutoModelForCausalLM, AutoModelForCausalLM, GPT2Config
|
3 |
import tensorflow as tf
|
4 |
|
5 |
+
task_specific_params = {
|
6 |
+
"text-generation": {
|
7 |
+
"do_sample": False,
|
8 |
+
"max_length": 50
|
9 |
+
}
|
10 |
+
}
|
11 |
+
|
12 |
+
config = GPT2Config.from_pretrained("Sentdex/GPyT", _name_or_path='prophetikai/code-gpt', use_cache=True, task_specific_params=task_specific_params)
|
13 |
tokenizer = AutoTokenizer.from_pretrained("Sentdex/GPyT")
|
14 |
+
tf_model = TFAutoModelForCausalLM.from_pretrained("Sentdex/GPyT", config=config)
|
15 |
+
pytorch_model = AutoModelForCausalLM.from_pretrained("Sentdex/GPyT", config=config)
|
16 |
|
17 |
config.save_pretrained('./')
|
18 |
tokenizer.save_pretrained(save_directory='./')
|
19 |
+
tf_model.save_pretrained(save_directory='./', saved_model=True, version='sentdex')
|
20 |
pytorch_model.save_pretrained(save_directory='./')
|