Update utils.py

utils.py

@@ -18,6 +18,7 @@ from pygments import highlight
 from pygments.lexers import guess_lexer,get_lexer_by_name
 from pygments.formatters import HtmlFormatter
 import transformers
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import datasets
 from datasets import load_dataset
 
@@ -73,7 +74,7 @@ def load_tokenizer_and_model(base_model,load_8bit=False):
     else:
         device = "cpu"
 
-    tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = False)
+    tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True)
     if device == "cuda":
         model = AutoModelForCausalLM.from_pretrained(
             base_model,