Maykeye
commited on
Commit
•
46068d3
1
Parent(s):
4d5aebb
Fixed tokenizer conversion
Browse filesOn my cloud GPU tokenizer loads fine, so catch doesn't trigger
and except block isn't executed. So we need to save it in try-block
as well
- train.ipynb +3 -1
train.ipynb
CHANGED
@@ -161,7 +161,9 @@
|
|
161 |
"if not Path('tokenizer.json').exists(): \n",
|
162 |
" try:\n",
|
163 |
" tokenizer = AutoTokenizer.from_pretrained(\"openlm-research/open_llama_3b\")\n",
|
164 |
-
"
|
|
|
|
|
165 |
" os.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"]=\"python\" \n",
|
166 |
" tokenizer = AutoTokenizer.from_pretrained(\"openlm-research/open_llama_3b\")\n",
|
167 |
" tokenizer.save_pretrained(\".\")\n",
|
|
|
161 |
"if not Path('tokenizer.json').exists(): \n",
|
162 |
" try:\n",
|
163 |
" tokenizer = AutoTokenizer.from_pretrained(\"openlm-research/open_llama_3b\")\n",
|
164 |
+
" tokenizer.save_pretrained(\".\")\n",
|
165 |
+
" except Exception as e:\n",
|
166 |
+
" print(e)\n",
|
167 |
" os.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"]=\"python\" \n",
|
168 |
" tokenizer = AutoTokenizer.from_pretrained(\"openlm-research/open_llama_3b\")\n",
|
169 |
" tokenizer.save_pretrained(\".\")\n",
|