Update utils.py

utils.py

@@ -54,32 +54,6 @@ def generate_prompt_with_history(text, history, tokenizer, max_length=2048):
     else:
         return None
 
-# Few-Shot Training wird durch ein spezielles Modell-Laden gemacht (setFit)
-def load_tokenizer_and_model_setFit(base_model, load_8bit=False):
-    if torch.cuda.is_available():
-        device = "cuda"
-    else:
-        device = "cpu"
-    
-    
-    #tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
-    tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
-    tokenizer.pad_token = tokenizer.eos_token
-    tokenizer.pad_token_id = tokenizer.eos_token_id
-    if device == "cuda":
-        model = SetFitModel.from_pretrained(
-            base_model,
-            load_in_8bit=load_8bit,
-            torch_dtype=torch.float16,
-            device_map="auto",
-            use_auth_token=True,
-        )
-    else:
-        model = SetFitModel.from_pretrained(
-            base_model, device_map={"": device}, low_cpu_mem_usage=True
-        )
-
-    return tokenizer,model, device
 
 
 def load_tokenizer_and_model(base_model, load_8bit=False):